From 8b993d0985f910f9a1db90d15817fbd9ad88e27b Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Mon, 9 Dec 2024 16:46:50 -0800 Subject: [PATCH 01/62] restruction OpenSearchSinkConfiguration to use openSearchSinkConfig instead of PluginSettings. Rebuild readConnectionConfiguration Signed-off-by: Maxwell Brown --- .../sink/opensearch/OpenSearchSinkIT.java | 6 +- .../opensearch/ConnectionConfiguration.java | 187 ++++++++++++------ .../sink/opensearch/OpenSearchSink.java | 7 +- .../sink/opensearch/OpenSearchSinkConfig.java | 86 ++++++++ .../OpenSearchSinkConfiguration.java | 24 ++- .../opensearch/configuration/AuthConfig.java | 30 --- .../OpenSearchClientRefresherTest.java | 2 +- 7 files changed, 237 insertions(+), 105 deletions(-) create mode 100644 data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfig.java delete mode 100644 data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/AuthConfig.java diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index d97feb30ba..5b463c793f 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -101,12 +101,12 @@ import static org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchIntegrationHelper.isOSBundle; import static org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchIntegrationHelper.waitForClusterStateUpdatesToFinish; import static org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchIntegrationHelper.wipeAllTemplates; -import static org.opensearch.dataprepper.plugins.source.opensearch.configuration.AuthConfig.AUTHENTICATION; -import static org.opensearch.dataprepper.plugins.source.opensearch.configuration.AuthConfig.PASSWORD; -import static org.opensearch.dataprepper.plugins.source.opensearch.configuration.AuthConfig.USERNAME; public class OpenSearchSinkIT { private static final int LUCENE_CHAR_LENGTH_LIMIT = 32_766; + private static final String AUTHENTICATION = "authentication"; + private static final String USERNAME = "username"; + private static final String PASSWORD = "password"; private static final String TEST_STRING_WITH_SPECIAL_CHARS = "Hello! Data-Prepper? #Example123"; private static final String TEST_STRING_WITH_NON_LATIN_CHARS = "Привет,Γειά σας,こんにちは,你好"; private static final String PLUGIN_NAME = "opensearch"; diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java index 7557c0390d..5c5a18e276 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java @@ -6,6 +6,7 @@ package org.opensearch.dataprepper.plugins.sink.opensearch; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.http.ConnectionClosedException; import org.apache.http.HttpHost; import org.apache.http.HttpRequestInterceptor; import org.apache.http.auth.AuthScope; @@ -31,7 +32,9 @@ import org.opensearch.dataprepper.aws.api.AwsRequestSigningApache4Interceptor; import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.PreSerializedJsonpMapper; -import org.opensearch.dataprepper.plugins.source.opensearch.configuration.AuthConfig; +import org.opensearch.dataprepper.plugins.source.opensearch.AuthConfig; +import org.opensearch.dataprepper.plugins.source.opensearch.configuration.AwsAuthenticationConfiguration; +import org.opensearch.dataprepper.plugins.source.opensearch.configuration.ServerlessOptions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.arns.Arn; @@ -112,7 +115,6 @@ public class ConnectionConfiguration { private final String awsStsExternalId; private final Map awsStsHeaderOverrides; private final Optional proxy; - private final String pipelineName; private final boolean serverless; private final String serverlessNetworkPolicyName; private final String serverlessCollectionName; @@ -203,23 +205,16 @@ private ConnectionConfiguration(final Builder builder) { this.serverlessCollectionName = builder.serverlessCollectionName; this.serverlessVpceId = builder.serverlessVpceId; this.requestCompressionEnabled = builder.requestCompressionEnabled; - this.pipelineName = builder.pipelineName; this.authConfig = builder.authConfig; } - public static ConnectionConfiguration readConnectionConfiguration(final PluginSetting pluginSetting){ - @SuppressWarnings("unchecked") - final List hosts = (List) pluginSetting.getAttributeFromSettings(HOSTS); + public static ConnectionConfiguration readConnectionConfiguration(final OpenSearchSinkConfig openSearchSinkConfig){ + final List hosts = openSearchSinkConfig.getHosts(); ConnectionConfiguration.Builder builder = new ConnectionConfiguration.Builder(hosts); - final String username = (String) pluginSetting.getAttributeFromSettings(USERNAME); - final String password = (String) pluginSetting.getAttributeFromSettings(PASSWORD); - builder.withPipelineName(pluginSetting.getPipelineName()); - final AuthConfig authConfig = AuthConfig.readAuthConfig(pluginSetting); + final String username = openSearchSinkConfig.getUsername(); + final String password = openSearchSinkConfig.getPassword(); + final AuthConfig authConfig = openSearchSinkConfig.getAuthConfig(); if (authConfig != null) { - if (username != null || password != null) { - throw new IllegalStateException("Deprecated username and password should not be set " + - "when authentication is configured."); - } builder = builder.withAuthConfig(authConfig); } else { if (username != null) { @@ -229,74 +224,149 @@ public static ConnectionConfiguration readConnectionConfiguration(final PluginSe builder = builder.withPassword(password); } } - final Integer socketTimeout = pluginSetting.getIntegerOrDefault(SOCKET_TIMEOUT, null); + final Integer socketTimeout = openSearchSinkConfig.getSocketTimeout(); if (socketTimeout != null) { builder = builder.withSocketTimeout(socketTimeout); } - final Integer connectTimeout = pluginSetting.getIntegerOrDefault(CONNECT_TIMEOUT, null); + final Integer connectTimeout = openSearchSinkConfig.getConnectTimeout(); if (connectTimeout != null) { builder = builder.withConnectTimeout(connectTimeout); } - Map awsOption = pluginSetting.getTypedMap(AWS_OPTION, String.class, Object.class); - boolean awsOptionUsed = false; builder.withAwsSigv4(false); - if (awsOption != null && !awsOption.isEmpty()) { - awsOptionUsed = true; - builder.withAwsSigv4(true); - builder.withAwsRegion((String)(awsOption.getOrDefault(AWS_REGION.substring(4), DEFAULT_AWS_REGION))); - builder.withAWSStsRoleArn((String)(awsOption.getOrDefault(AWS_STS_ROLE_ARN.substring(4), null))); - builder.withAWSStsExternalId((String)(awsOption.getOrDefault(AWS_STS_EXTERNAL_ID.substring(4), null))); - builder.withAwsStsHeaderOverrides((Map)awsOption.get(AWS_STS_HEADER_OVERRIDES.substring(4))); - builder.withServerless(OBJECT_MAPPER.convertValue( - awsOption.getOrDefault(SERVERLESS, false), Boolean.class)); - - Map serverlessOptions = (Map) awsOption.get(SERVERLESS_OPTIONS); - if (serverlessOptions != null && !serverlessOptions.isEmpty()) { - builder.withServerlessNetworkPolicyName((String)(serverlessOptions.getOrDefault(NETWORK_POLICY_NAME, null))); - builder.withServerlessCollectionName((String)(serverlessOptions.getOrDefault(COLLECTION_NAME, null))); - builder.withServerlessVpceId((String)(serverlessOptions.getOrDefault(VPCE_ID, null))); - } + final AwsAuthenticationConfiguration awsAuthenticationConfiguration = openSearchSinkConfig.getAwsAuthenticationOptions(); + if (awsAuthenticationConfiguration != null) { + builder = builder.withAwsSigv4(true) + .withAwsRegion(awsAuthenticationConfiguration.getAwsRegion().toString()) + .withAWSStsRoleArn(awsAuthenticationConfiguration.getAwsStsRoleArn()) + .withAWSStsExternalId(awsAuthenticationConfiguration.getAwsStsExternalId()) + .withAwsStsHeaderOverrides(awsAuthenticationConfiguration.getAwsStsHeaderOverrides()) + .withServerless(awsAuthenticationConfiguration.isServerlessCollection()); + + final ServerlessOptions serverlessOptions = awsAuthenticationConfiguration.getServerlessOptions(); + if (serverlessOptions != null) { + builder = builder.withServerlessNetworkPolicyName(serverlessOptions.getNetworkPolicyName()) + .withServerlessCollectionName(serverlessOptions.getCollectionName()) + .withServerlessVpceId(serverlessOptions.getVpceId()); + } } else { builder.withServerless(false); } - boolean awsSigv4 = pluginSetting.getBooleanOrDefault(AWS_SIGV4, false); - final String awsOptionConflictMessage = String.format("%s option cannot be used along with %s option", AWS_SIGV4, AWS_OPTION); - if (awsSigv4) { - if (awsOptionUsed) { - throw new RuntimeException(awsOptionConflictMessage); - } - builder.withAwsSigv4(true); - builder.withAwsRegion(pluginSetting.getStringOrDefault(AWS_REGION, DEFAULT_AWS_REGION)); - builder.withAWSStsRoleArn(pluginSetting.getStringOrDefault(AWS_STS_ROLE_ARN, null)); - builder.withAWSStsExternalId(pluginSetting.getStringOrDefault(AWS_STS_EXTERNAL_ID, null)); - builder.withAwsStsHeaderOverrides(pluginSetting.getTypedMap(AWS_STS_HEADER_OVERRIDES, String.class, String.class)); - } - final String certPath = pluginSetting.getStringOrDefault(CERT_PATH, null); - final boolean insecure = pluginSetting.getBooleanOrDefault(INSECURE, false); + final String certPath = openSearchSinkConfig.getCertPath(); + final boolean insecure = openSearchSinkConfig.getInsecure(); if (certPath != null) { builder = builder.withCert(certPath); } else { //We will set insecure flag only if certPath is null builder = builder.withInsecure(insecure); } - final String proxy = pluginSetting.getStringOrDefault(PROXY, null); - builder = builder.withProxy(proxy); + final String proxy = openSearchSinkConfig.getProxy(); + if (proxy != null) { + builder = builder.withProxy(proxy); + } - final String distributionVersionName = pluginSetting.getStringOrDefault(DISTRIBUTION_VERSION, - DistributionVersion.DEFAULT.getVersion()); + final String distributionVersionName = openSearchSinkConfig.getDistributionVersion(); final DistributionVersion distributionVersion = DistributionVersion.fromTypeName(distributionVersionName); - final boolean requestCompressionEnabled = pluginSetting.getBooleanOrDefault( - REQUEST_COMPRESSION_ENABLED, !DistributionVersion.ES6.equals(distributionVersion)); + final boolean requestCompressionEnabled = openSearchSinkConfig.getEnableRequestCompression(!DistributionVersion.ES6.equals(distributionVersion)); builder = builder.withRequestCompressionEnabled(requestCompressionEnabled); return builder.build(); } - public String getPipelineName() { - return pipelineName; + public static ConnectionConfiguration readConnectionConfiguration(final PluginSetting pluginSetting){ + //Must fix all calls to this function + final List hosts = (List) pluginSetting.getAttributeFromSettings(HOSTS); + ConnectionConfiguration.Builder builder = new ConnectionConfiguration.Builder(hosts); + return builder.build(); } +// +// public static ConnectionConfiguration readConnectionConfiguration(final PluginSetting pluginSetting){ +// @SuppressWarnings("unchecked") +// final List hosts = (List) pluginSetting.getAttributeFromSettings(HOSTS); +// ConnectionConfiguration.Builder builder = new ConnectionConfiguration.Builder(hosts); +// final String username = (String) pluginSetting.getAttributeFromSettings(USERNAME); +// final String password = (String) pluginSetting.getAttributeFromSettings(PASSWORD); +// builder.withPipelineName(pluginSetting.getPipelineName()); +// final AuthConfig authConfig = AuthConfig.readAuthConfig(pluginSetting); +// if (authConfig != null) { +// if (username != null || password != null) { +// throw new IllegalStateException("Deprecated username and password should not be set " + +// "when authentication is configured."); +// } +// builder = builder.withAuthConfig(authConfig); +// } else { +// if (username != null) { +// builder = builder.withUsername(username); +// } +// if (password != null) { +// builder = builder.withPassword(password); +// } +// } +// final Integer socketTimeout = pluginSetting.getIntegerOrDefault(SOCKET_TIMEOUT, null); +// if (socketTimeout != null) { +// builder = builder.withSocketTimeout(socketTimeout); +// } +// final Integer connectTimeout = pluginSetting.getIntegerOrDefault(CONNECT_TIMEOUT, null); +// if (connectTimeout != null) { +// builder = builder.withConnectTimeout(connectTimeout); +// } +// +// Map awsOption = pluginSetting.getTypedMap(AWS_OPTION, String.class, Object.class); +// boolean awsOptionUsed = false; +// builder.withAwsSigv4(false); +// if (awsOption != null && !awsOption.isEmpty()) { +// awsOptionUsed = true; +// builder.withAwsSigv4(true); +// builder.withAwsRegion((String)(awsOption.getOrDefault(AWS_REGION.substring(4), DEFAULT_AWS_REGION))); +// builder.withAWSStsRoleArn((String)(awsOption.getOrDefault(AWS_STS_ROLE_ARN.substring(4), null))); +// builder.withAWSStsExternalId((String)(awsOption.getOrDefault(AWS_STS_EXTERNAL_ID.substring(4), null))); +// builder.withAwsStsHeaderOverrides((Map)awsOption.get(AWS_STS_HEADER_OVERRIDES.substring(4))); +// builder.withServerless(OBJECT_MAPPER.convertValue( +// awsOption.getOrDefault(SERVERLESS, false), Boolean.class)); +// +// Map serverlessOptions = (Map) awsOption.get(SERVERLESS_OPTIONS); +// if (serverlessOptions != null && !serverlessOptions.isEmpty()) { +// builder.withServerlessNetworkPolicyName((String)(serverlessOptions.getOrDefault(NETWORK_POLICY_NAME, null))); +// builder.withServerlessCollectionName((String)(serverlessOptions.getOrDefault(COLLECTION_NAME, null))); +// builder.withServerlessVpceId((String)(serverlessOptions.getOrDefault(VPCE_ID, null))); +// } +// } else { +// builder.withServerless(false); +// } +// boolean awsSigv4 = pluginSetting.getBooleanOrDefault(AWS_SIGV4, false); +// final String awsOptionConflictMessage = String.format("%s option cannot be used along with %s option", AWS_SIGV4, AWS_OPTION); +// if (awsSigv4) { +// if (awsOptionUsed) { +// throw new RuntimeException(awsOptionConflictMessage); +// } +// builder.withAwsSigv4(true); +// builder.withAwsRegion(pluginSetting.getStringOrDefault(AWS_REGION, DEFAULT_AWS_REGION)); +// builder.withAWSStsRoleArn(pluginSetting.getStringOrDefault(AWS_STS_ROLE_ARN, null)); +// builder.withAWSStsExternalId(pluginSetting.getStringOrDefault(AWS_STS_EXTERNAL_ID, null)); +// builder.withAwsStsHeaderOverrides(pluginSetting.getTypedMap(AWS_STS_HEADER_OVERRIDES, String.class, String.class)); +// } +// +// final String certPath = pluginSetting.getStringOrDefault(CERT_PATH, null); +// final boolean insecure = pluginSetting.getBooleanOrDefault(INSECURE, false); +// if (certPath != null) { +// builder = builder.withCert(certPath); +// } else { +// //We will set insecure flag only if certPath is null +// builder = builder.withInsecure(insecure); +// } +// final String proxy = pluginSetting.getStringOrDefault(PROXY, null); +// builder = builder.withProxy(proxy); +// +// final String distributionVersionName = pluginSetting.getStringOrDefault(DISTRIBUTION_VERSION, +// DistributionVersion.DEFAULT.getVersion()); +// final DistributionVersion distributionVersion = DistributionVersion.fromTypeName(distributionVersionName); +// final boolean requestCompressionEnabled = pluginSetting.getBooleanOrDefault( +// REQUEST_COMPRESSION_ENABLED, !DistributionVersion.ES6.equals(distributionVersion)); +// builder = builder.withRequestCompressionEnabled(requestCompressionEnabled); +// +// return builder.build(); +// } public RestHighLevelClient createClient(AwsCredentialsSupplier awsCredentialsSupplier) { final HttpHost[] httpHosts = new HttpHost[hosts.size()]; @@ -632,11 +702,6 @@ public Builder withProxy(final String proxy) { return this; } - public Builder withPipelineName(final String pipelineName) { - this.pipelineName = pipelineName; - return this; - } - public Builder withServerless(boolean serverless) { this.serverless = serverless; return this; diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java index 2248ba669a..98702612d1 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java @@ -86,7 +86,7 @@ import java.util.function.Supplier; import java.util.stream.Collectors; -@DataPrepperPlugin(name = "opensearch", pluginType = Sink.class) +@DataPrepperPlugin(name = "opensearch", pluginType = Sink.class, pluginConfigurationType = OpenSearchSinkConfig.class) public class OpenSearchSink extends AbstractSink> { public static final String BULKREQUEST_LATENCY = "bulkRequestLatency"; public static final String BULKREQUEST_ERRORS = "bulkRequestErrors"; @@ -151,7 +151,8 @@ public OpenSearchSink(final PluginSetting pluginSetting, final SinkContext sinkContext, final ExpressionEvaluator expressionEvaluator, final AwsCredentialsSupplier awsCredentialsSupplier, - final PluginConfigObservable pluginConfigObservable) { + final PluginConfigObservable pluginConfigObservable, + final OpenSearchSinkConfig openSearchSinkConfiguration) { super(pluginSetting, Integer.MAX_VALUE, INITIALIZE_RETRY_WAIT_TIME_MS); this.awsCredentialsSupplier = awsCredentialsSupplier; this.sinkContext = sinkContext != null ? sinkContext : new SinkContext(null, Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); @@ -163,7 +164,7 @@ public OpenSearchSink(final PluginSetting pluginSetting, bulkRequestSizeBytesSummary = pluginMetrics.summary(BULKREQUEST_SIZE_BYTES); dynamicDocumentVersionDroppedEvents = pluginMetrics.counter(INVALID_VERSION_EXPRESSION_DROPPED_EVENTS); - this.openSearchSinkConfig = OpenSearchSinkConfiguration.readESConfig(pluginSetting, expressionEvaluator); + this.openSearchSinkConfig = OpenSearchSinkConfiguration.readOSConfig(openSearchSinkConfiguration); this.bulkSize = ByteSizeUnit.MB.toBytes(openSearchSinkConfig.getIndexConfiguration().getBulkSize()); this.flushTimeout = openSearchSinkConfig.getIndexConfiguration().getFlushTimeout(); this.indexType = openSearchSinkConfig.getIndexConfiguration().getIndexType(); diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfig.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfig.java new file mode 100644 index 0000000000..f3a6daa47e --- /dev/null +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfig.java @@ -0,0 +1,86 @@ +package org.opensearch.dataprepper.plugins.sink.opensearch; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.linecorp.armeria.server.annotation.Get; +import jakarta.validation.Valid; +import jakarta.validation.constraints.AssertTrue; +import lombok.Getter; +import org.opensearch.dataprepper.plugins.source.opensearch.AuthConfig; +import org.opensearch.dataprepper.plugins.source.opensearch.configuration.AwsAuthenticationConfiguration; + +import java.util.List; +import java.util.Objects; + +public class OpenSearchSinkConfig { + @Getter + @JsonProperty("hosts") + private List hosts; + + @Getter + @JsonProperty("username") + private String username = null; + + @Getter + @JsonProperty("password") + private String password = null; + + @Getter + @JsonProperty("authconfig") + private AuthConfig authConfig = null; + + @Getter + @JsonProperty("socket_timeout") + private Integer socketTimeout = null; + + @Getter + @JsonProperty("connect_timeout") + private Integer connectTimeout = null; + + @Getter + @JsonProperty("aws") + @Valid + private AwsAuthenticationConfiguration awsAuthenticationOptions; + + @Getter + @Deprecated + @JsonProperty("aws_sigv4") + private Boolean awsSigv4 = false; + + @Getter + @JsonProperty("cert") + private String certPath = null; + + @Getter + @JsonProperty("insecure") + private Boolean insecure = false; + + @Getter + @JsonProperty("proxy") + private String proxy = null; + + @Getter + @JsonProperty("distribution_version") + private String distributionVersion = DistributionVersion.DEFAULT.getVersion(); + + @JsonProperty("enable_request_compression") + private Boolean enableRequestCompression = null; + + public Boolean getEnableRequestCompression(boolean defaultValue) { + return Objects.requireNonNullElse(enableRequestCompression, defaultValue); + } + + public void validateConfig() { + isValidAuthConfig(); + } + + + void isValidAuthConfig() { + if (authConfig != null) { + if (username != null || password != null) { + throw new IllegalStateException("Deprecated username and password should not be set " + + "when authentication is configured."); + } + } + } +} + diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfiguration.java index 4ec8f17bc8..6e4bd513c4 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfiguration.java @@ -40,15 +40,25 @@ private OpenSearchSinkConfiguration( this.retryConfiguration = retryConfiguration; } - public static OpenSearchSinkConfiguration readESConfig(final PluginSetting pluginSetting) { - return readESConfig(pluginSetting, null); - } +// public static OpenSearchSinkConfiguration readESConfig(final PluginSetting pluginSetting) { +// return readESConfig(pluginSetting, null); +// } +// +// public static OpenSearchSinkConfiguration readESConfig(final PluginSetting pluginSetting, final ExpressionEvaluator expressionEvaluator) { +// final ConnectionConfiguration connectionConfiguration = +// ConnectionConfiguration.readConnectionConfiguration(pluginSetting); +// final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting, expressionEvaluator); +// final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(pluginSetting); +// +// return new OpenSearchSinkConfiguration(connectionConfiguration, indexConfiguration, retryConfiguration); +// } - public static OpenSearchSinkConfiguration readESConfig(final PluginSetting pluginSetting, final ExpressionEvaluator expressionEvaluator) { + public static OpenSearchSinkConfiguration readOSConfig(final OpenSearchSinkConfig openSearchSinkConfig) { + openSearchSinkConfig.validateConfig(); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting, expressionEvaluator); - final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); + final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig); + final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(openSearchSinkConfig); return new OpenSearchSinkConfiguration(connectionConfiguration, indexConfiguration, retryConfiguration); } diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/AuthConfig.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/AuthConfig.java deleted file mode 100644 index 8893535eb9..0000000000 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/AuthConfig.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.opensearch.dataprepper.plugins.source.opensearch.configuration; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.opensearch.dataprepper.model.configuration.PluginSetting; - -import java.util.Map; - -public class AuthConfig { - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - public static final String AUTHENTICATION = "authentication"; - public static final String USERNAME = "username"; - public static final String PASSWORD = "password"; - private String username; - - private String password; - - public String getUsername() { - return username; - } - - public String getPassword() { - return password; - } - - public static AuthConfig readAuthConfig(final PluginSetting pluginSetting) { - final Map authConfigMap = - pluginSetting.getTypedMap(AUTHENTICATION, String.class, Object.class); - return authConfigMap.isEmpty() ? null : OBJECT_MAPPER.convertValue(authConfigMap, AuthConfig.class); - } -} diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresherTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresherTest.java index 584051dff6..e879d47396 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresherTest.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresherTest.java @@ -10,7 +10,7 @@ import org.opensearch.client.opensearch.OpenSearchClient; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.configuration.PluginSetting; -import org.opensearch.dataprepper.plugins.source.opensearch.configuration.AuthConfig; +import org.opensearch.dataprepper.plugins.source.opensearch.AuthConfig; import java.util.function.Function; From 23afcbf266ed26681900e6b2ec0a2bb8cf04fbd5 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Tue, 10 Dec 2024 16:12:02 -0800 Subject: [PATCH 02/62] rebuild readIndexConfig and associated changes Signed-off-by: Maxwell Brown --- .../opensearch/ConnectionConfiguration.java | 17 +- .../sink/opensearch/OpenSearchSink.java | 15 +- .../sink/opensearch/OpenSearchSinkConfig.java | 123 ++++++-- .../OpenSearchSinkConfiguration.java | 4 +- .../configuration/ActionConfiguration.java | 23 ++ .../opensearch/index/IndexConfiguration.java | 285 ++++++++++++------ .../sink/opensearch/index/TemplateType.java | 2 +- 7 files changed, 329 insertions(+), 140 deletions(-) create mode 100644 data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ActionConfiguration.java diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java index 5c5a18e276..e76d9189a0 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java @@ -6,7 +6,6 @@ package org.opensearch.dataprepper.plugins.sink.opensearch; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.http.ConnectionClosedException; import org.apache.http.HttpHost; import org.apache.http.HttpRequestInterceptor; import org.apache.http.auth.AuthScope; @@ -64,7 +63,6 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; -import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.DISTRIBUTION_VERSION; public class ConnectionConfiguration { static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); @@ -213,16 +211,11 @@ public static ConnectionConfiguration readConnectionConfiguration(final OpenSear ConnectionConfiguration.Builder builder = new ConnectionConfiguration.Builder(hosts); final String username = openSearchSinkConfig.getUsername(); final String password = openSearchSinkConfig.getPassword(); - final AuthConfig authConfig = openSearchSinkConfig.getAuthConfig(); - if (authConfig != null) { - builder = builder.withAuthConfig(authConfig); - } else { - if (username != null) { - builder = builder.withUsername(username); - } - if (password != null) { - builder = builder.withPassword(password); - } + if (username != null) { + builder = builder.withUsername(username); + } + if (password != null) { + builder = builder.withPassword(password); } final Integer socketTimeout = openSearchSinkConfig.getSocketTimeout(); if (socketTimeout != null) { diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java index 98702612d1..b58e8165c5 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java @@ -28,6 +28,7 @@ import org.opensearch.dataprepper.metrics.MetricNames; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; +import org.opensearch.dataprepper.model.configuration.PipelineDescription; import org.opensearch.dataprepper.model.configuration.PluginModel; import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.event.Event; @@ -53,6 +54,7 @@ import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.JavaClientAccumulatingCompressedBulkRequest; import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.JavaClientAccumulatingUncompressedBulkRequest; import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.SerializedJson; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.ActionConfiguration; import org.opensearch.dataprepper.plugins.sink.opensearch.dlq.FailedBulkOperation; import org.opensearch.dataprepper.plugins.sink.opensearch.dlq.FailedBulkOperationConverter; import org.opensearch.dataprepper.plugins.sink.opensearch.dlq.FailedDlqData; @@ -117,7 +119,7 @@ public class OpenSearchSink extends AbstractSink> { private final String routing; private final String pipeline; private final String action; - private final List> actions; + private final List actions; private final String documentRootKey; private String configuredIndexAlias; private final ReentrantLock lock; @@ -151,12 +153,14 @@ public OpenSearchSink(final PluginSetting pluginSetting, final SinkContext sinkContext, final ExpressionEvaluator expressionEvaluator, final AwsCredentialsSupplier awsCredentialsSupplier, + final PipelineDescription pipelineDescription, final PluginConfigObservable pluginConfigObservable, final OpenSearchSinkConfig openSearchSinkConfiguration) { super(pluginSetting, Integer.MAX_VALUE, INITIALIZE_RETRY_WAIT_TIME_MS); this.awsCredentialsSupplier = awsCredentialsSupplier; this.sinkContext = sinkContext != null ? sinkContext : new SinkContext(null, Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); this.expressionEvaluator = expressionEvaluator; + this.pipeline = pipelineDescription.getPipelineName(); bulkRequestTimer = pluginMetrics.timer(BULKREQUEST_LATENCY); bulkRequestErrorsCounter = pluginMetrics.counter(BULKREQUEST_ERRORS); invalidActionErrorsCounter = pluginMetrics.counter(INVALID_ACTION_ERRORS); @@ -164,7 +168,7 @@ public OpenSearchSink(final PluginSetting pluginSetting, bulkRequestSizeBytesSummary = pluginMetrics.summary(BULKREQUEST_SIZE_BYTES); dynamicDocumentVersionDroppedEvents = pluginMetrics.counter(INVALID_VERSION_EXPRESSION_DROPPED_EVENTS); - this.openSearchSinkConfig = OpenSearchSinkConfiguration.readOSConfig(openSearchSinkConfiguration); + this.openSearchSinkConfig = OpenSearchSinkConfiguration.readOSConfig(openSearchSinkConfiguration, expressionEvaluator); this.bulkSize = ByteSizeUnit.MB.toBytes(openSearchSinkConfig.getIndexConfiguration().getBulkSize()); this.flushTimeout = openSearchSinkConfig.getIndexConfiguration().getFlushTimeout(); this.indexType = openSearchSinkConfig.getIndexConfiguration().getIndexType(); @@ -172,7 +176,6 @@ public OpenSearchSink(final PluginSetting pluginSetting, this.documentId = openSearchSinkConfig.getIndexConfiguration().getDocumentId(); this.routingField = openSearchSinkConfig.getIndexConfiguration().getRoutingField(); this.routing = openSearchSinkConfig.getIndexConfiguration().getRouting(); - this.pipeline = openSearchSinkConfig.getIndexConfiguration().getPipeline(); this.action = openSearchSinkConfig.getIndexConfiguration().getAction(); this.actions = openSearchSinkConfig.getIndexConfiguration().getActions(); this.documentRootKey = openSearchSinkConfig.getIndexConfiguration().getDocumentRootKey(); @@ -430,9 +433,9 @@ public void doOutput(final Collection> records) { String eventAction = action; if (actions != null) { - for (final Map actionEntry: actions) { - final String condition = (String)actionEntry.get("when"); - eventAction = (String)actionEntry.get("type"); + for (final ActionConfiguration actionEntry: actions) { + final String condition = actionEntry.getWhen(); + eventAction = actionEntry.getType(); if (condition != null && expressionEvaluator.evaluateConditional(condition, event)) { break; diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfig.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfig.java index f3a6daa47e..c7b9eb4ffd 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfig.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfig.java @@ -1,11 +1,11 @@ package org.opensearch.dataprepper.plugins.sink.opensearch; import com.fasterxml.jackson.annotation.JsonProperty; -import com.linecorp.armeria.server.annotation.Get; import jakarta.validation.Valid; -import jakarta.validation.constraints.AssertTrue; import lombok.Getter; -import org.opensearch.dataprepper.plugins.source.opensearch.AuthConfig; +import org.opensearch.dataprepper.model.opensearch.OpenSearchBulkActions; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.ActionConfiguration; +import org.opensearch.dataprepper.plugins.sink.opensearch.index.TemplateType; import org.opensearch.dataprepper.plugins.source.opensearch.configuration.AwsAuthenticationConfiguration; import java.util.List; @@ -24,10 +24,6 @@ public class OpenSearchSinkConfig { @JsonProperty("password") private String password = null; - @Getter - @JsonProperty("authconfig") - private AuthConfig authConfig = null; - @Getter @JsonProperty("socket_timeout") private Integer socketTimeout = null; @@ -41,11 +37,6 @@ public class OpenSearchSinkConfig { @Valid private AwsAuthenticationConfiguration awsAuthenticationOptions; - @Getter - @Deprecated - @JsonProperty("aws_sigv4") - private Boolean awsSigv4 = false; - @Getter @JsonProperty("cert") private String certPath = null; @@ -69,18 +60,112 @@ public Boolean getEnableRequestCompression(boolean defaultValue) { return Objects.requireNonNullElse(enableRequestCompression, defaultValue); } + @Getter + @JsonProperty("index") + private String indexAlias = null; + + @Getter + @JsonProperty("index_type") + private String indexType = null; + + @Getter + @JsonProperty("template_type") + private String templateType = TemplateType.V1.getTypeName(); + + @Getter + @JsonProperty("template_file") + private String templateFile = null; + + @Getter + @JsonProperty("template_content") + private String templateContent = null; + + @Getter + @JsonProperty("number_of_shards") + private Integer numShards = 0; + + @Getter + @JsonProperty("number_of_replicas") + private Integer numReplicas = 0; + + @JsonProperty("bulk_size") + private Long bulkSize; + + public Long getBulkSize(Long defaultBulkSize) { + return bulkSize == null ? defaultBulkSize : bulkSize; + } + + @JsonProperty("estimate_bulk_size_using_compression") + private Boolean estimateBulkSizeUsingCompression; + + public Boolean getEstimateBulkSizeUsingCompression(boolean defaultValue) { + return Objects.requireNonNullElse(estimateBulkSizeUsingCompression, defaultValue); + } + + @JsonProperty("max_local_compressions_for_estimation") + private Integer maxLocalCompressionsForEstimation; + + public Integer getMaxLocalCompressionsForEstimation(Integer defaultValue) { + return Objects.requireNonNullElse(maxLocalCompressionsForEstimation, defaultValue); + } + + @JsonProperty("flush_timeout") + private Long flushTimeout = null; + + public Long getFlushTimeout(Long defaultFlushTimeout) { + return flushTimeout == null ? defaultFlushTimeout : flushTimeout; + } + + @Getter + @JsonProperty("document_version_type") + private String versionType = null; + + @Getter + @JsonProperty("document_version") + private String versionExpression = null; + + @Getter + @JsonProperty("normalize_index") + private Boolean normalizeIndex = false; + + @Getter + @JsonProperty("document_id") + private String documentId = null; + + @Getter + @JsonProperty("routing") + private String routing = null; + + @Getter + @JsonProperty("ism_policy_file") + private String ismPolicyFile = null; + + @Getter + @JsonProperty("action") + private String action = OpenSearchBulkActions.INDEX.toString(); + + @Getter + @Valid + @JsonProperty("actions") + private List actions = null; + + @Getter + @JsonProperty("document_root_key") + private String documentRootKey = null; + + + public void validateConfig() { - isValidAuthConfig(); + isActionValid(); } - void isValidAuthConfig() { - if (authConfig != null) { - if (username != null || password != null) { - throw new IllegalStateException("Deprecated username and password should not be set " + - "when authentication is configured."); - } + void isActionValid() { + if (action.equals("index") || action.equals("create") || action.equals("update") || action.equals("upsert") || action.equals("delete")) { + return; } + throw new IllegalArgumentException("action must be one of [index, create, update, upsert, delete]"); } + } diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfiguration.java index 6e4bd513c4..b3bfe801ac 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfiguration.java @@ -53,11 +53,11 @@ private OpenSearchSinkConfiguration( // return new OpenSearchSinkConfiguration(connectionConfiguration, indexConfiguration, retryConfiguration); // } - public static OpenSearchSinkConfiguration readOSConfig(final OpenSearchSinkConfig openSearchSinkConfig) { + public static OpenSearchSinkConfiguration readOSConfig(final OpenSearchSinkConfig openSearchSinkConfig, final ExpressionEvaluator expressionEvaluator) { openSearchSinkConfig.validateConfig(); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig); + final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig, expressionEvaluator); final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(openSearchSinkConfig); return new OpenSearchSinkConfiguration(connectionConfiguration, indexConfiguration, retryConfiguration); diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ActionConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ActionConfiguration.java new file mode 100644 index 0000000000..8c53e0d87c --- /dev/null +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ActionConfiguration.java @@ -0,0 +1,23 @@ +package org.opensearch.dataprepper.plugins.sink.opensearch.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.AssertTrue; +import lombok.Getter; + +public class ActionConfiguration { + @Getter + @JsonProperty("type") + private String type; + + @Getter + @JsonProperty("when") + private String when; + + @AssertTrue(message = "type must be one of index, create, update, upsert, delete") + boolean isTypeValid() { + if (type.equals("index") || type.equals("create") || type.equals("update") || type.equals("upsert") || type.equals("delete")) { + return true; + } + return false; + } +} diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java index 10dd892296..4de7c844d7 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java @@ -14,14 +14,18 @@ import org.opensearch.dataprepper.model.opensearch.OpenSearchBulkActions; import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException; import org.opensearch.dataprepper.plugins.sink.opensearch.DistributionVersion; +import org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchSinkConfig; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.ActionConfiguration; import org.opensearch.dataprepper.plugins.sink.opensearch.s3.FileReader; import org.opensearch.dataprepper.plugins.sink.opensearch.s3.S3ClientProvider; import org.opensearch.dataprepper.plugins.sink.opensearch.s3.S3FileReader; +import org.opensearch.dataprepper.plugins.source.opensearch.configuration.AwsAuthenticationConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.arns.Arn; import software.amazon.awssdk.services.s3.S3Client; +import javax.swing.*; import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -82,7 +86,6 @@ public class IndexConfiguration { private final Map indexTemplate; private final String documentIdField; private final String documentId; - private final String pipeline; private final String routingField; private final String routing; private final long bulkSize; @@ -91,7 +94,7 @@ public class IndexConfiguration { private final long flushTimeout; private final Optional ismPolicyFile; private final String action; - private final List> actions; + private final List actions; private final String s3AwsRegion; private final String s3AwsStsRoleArn; private final String s3AwsExternalId; @@ -149,7 +152,6 @@ private IndexConfiguration(final Builder builder) { this.flushTimeout = builder.flushTimeout; this.routingField = builder.routingField; this.routing = builder.routing; - this.pipeline = builder.pipeline; String documentIdField = builder.documentIdField; String documentId = builder.documentId; @@ -188,133 +190,225 @@ private void determineTemplateType(Builder builder) { } } - public static IndexConfiguration readIndexConfig(final PluginSetting pluginSetting) { - return readIndexConfig(pluginSetting, null); + public static IndexConfiguration readIndexConfig(final OpenSearchSinkConfig openSearchSinkConfig) { + return readIndexConfig(openSearchSinkConfig, null); } - public static IndexConfiguration readIndexConfig(final PluginSetting pluginSetting, final ExpressionEvaluator expressionEvaluator) { + public static IndexConfiguration readIndexConfig(final OpenSearchSinkConfig openSearchSinkConfig, final ExpressionEvaluator expressionEvaluator) { IndexConfiguration.Builder builder = new IndexConfiguration.Builder(); - final String indexAlias = pluginSetting.getStringOrDefault(INDEX_ALIAS, null); + final String indexAlias = openSearchSinkConfig.getIndexAlias(); if (indexAlias != null) { builder = builder.withIndexAlias(indexAlias); } - final String indexType = pluginSetting.getStringOrDefault(INDEX_TYPE, null); - if(indexType != null) { + final String indexType = openSearchSinkConfig.getIndexType(); + if (indexType != null) { builder = builder.withIndexType(indexType); } - final String templateType = pluginSetting.getStringOrDefault(TEMPLATE_TYPE, TemplateType.V1.getTypeName()); - if(templateType != null) { + final String templateType = openSearchSinkConfig.getTemplateType(); + if (templateType != null) { builder = builder.withTemplateType(templateType); } - final String templateFile = pluginSetting.getStringOrDefault(TEMPLATE_FILE, null); + final String templateFile = openSearchSinkConfig.getTemplateFile(); if (templateFile != null) { builder = builder.withTemplateFile(templateFile); } - - final String templateContent = pluginSetting.getStringOrDefault(TEMPLATE_CONTENT, null); + final String templateContent = openSearchSinkConfig.getTemplateContent(); if (templateContent != null) { builder = builder.withTemplateContent(templateContent); } - if (templateContent != null && templateFile != null) { LOG.warn("Both template_content and template_file are configured. Only template_content will be used"); } + final String documentId = openSearchSinkConfig.getDocumentId(); + if (documentId != null) { + builder = builder.withDocumentId(documentId); + } + final String routing = openSearchSinkConfig.getRouting(); + if (routing != null) { + builder = builder.withRouting(routing); + } - builder = builder.withNumShards(pluginSetting.getIntegerOrDefault(NUM_SHARDS, 0)); - builder = builder.withNumReplicas(pluginSetting.getIntegerOrDefault(NUM_REPLICAS, 0)); - final Long batchSize = pluginSetting.getLongOrDefault(BULK_SIZE, DEFAULT_BULK_SIZE); - builder = builder.withBulkSize(batchSize); - final boolean estimateBulkSizeUsingCompression = - pluginSetting.getBooleanOrDefault(ESTIMATE_BULK_SIZE_USING_COMPRESSION, DEFAULT_ESTIMATE_BULK_SIZE_USING_COMPRESSION); - builder = builder.withEstimateBulkSizeUsingCompression(estimateBulkSizeUsingCompression); - - final int maxLocalCompressionsForEstimation = - pluginSetting.getIntegerOrDefault(MAX_LOCAL_COMPRESSIONS_FOR_ESTIMATION, DEFAULT_MAX_LOCAL_COMPRESSIONS_FOR_ESTIMATION); - builder = builder.withMaxLocalCompressionsForEstimation(maxLocalCompressionsForEstimation); - - final long flushTimeout = pluginSetting.getLongOrDefault(FLUSH_TIMEOUT, DEFAULT_FLUSH_TIMEOUT); - builder = builder.withFlushTimeout(flushTimeout); - final String documentIdField = pluginSetting.getStringOrDefault(DOCUMENT_ID_FIELD, null); - final String documentId = pluginSetting.getStringOrDefault(DOCUMENT_ID, null); + builder = builder.withNumShards(openSearchSinkConfig.getNumShards()) + .withNumReplicas(openSearchSinkConfig.getNumReplicas()) + .withBulkSize(openSearchSinkConfig.getBulkSize(DEFAULT_BULK_SIZE)) + .withEstimateBulkSizeUsingCompression(openSearchSinkConfig.getEstimateBulkSizeUsingCompression(DEFAULT_ESTIMATE_BULK_SIZE_USING_COMPRESSION)) + .withMaxLocalCompressionsForEstimation(openSearchSinkConfig.getMaxLocalCompressionsForEstimation(DEFAULT_MAX_LOCAL_COMPRESSIONS_FOR_ESTIMATION)) + .withFlushTimeout(openSearchSinkConfig.getFlushTimeout(DEFAULT_FLUSH_TIMEOUT)) + .withVersionType(openSearchSinkConfig.getVersionType()) + .withNormalizeIndex(openSearchSinkConfig.getNormalizeIndex()) + .withIsmPolicyFile(openSearchSinkConfig.getIsmPolicyFile()) + .withDocumentRootKey(openSearchSinkConfig.getDocumentRootKey()) + .withDistributionVersion(openSearchSinkConfig.getDistributionVersion()); - final String versionExpression = pluginSetting.getStringOrDefault(DOCUMENT_VERSION_EXPRESSION, null); - final String versionType = pluginSetting.getStringOrDefault(DOCUMENT_VERSION_TYPE, null); - final boolean normalizeIndex = pluginSetting.getBooleanOrDefault(NORMALIZE_INDEX, false); - builder = builder.withNormalizeIndex(normalizeIndex); + final String versionExpression = openSearchSinkConfig.getVersionExpression(); builder = builder.withVersionExpression(versionExpression); if (versionExpression != null && (!expressionEvaluator.isValidFormatExpression(versionExpression))) { throw new InvalidPluginConfigurationException( String.format("document_version \"%s\" is not a valid format expression.", versionExpression)); } - builder = builder.withVersionType(versionType); - - if (Objects.nonNull(documentIdField) && Objects.nonNull(documentId)) { - throw new InvalidPluginConfigurationException("Both document_id_field and document_id cannot be used at the same time. It is preferred to only use document_id as document_id_field is deprecated."); - } - - if (documentIdField != null) { - LOG.warn("document_id_field is deprecated in favor of document_id, and support for document_id_field will be removed in a future major version release."); - builder = builder.withDocumentIdField(documentIdField); - } else if (documentId != null) { - builder = builder.withDocumentId(documentId); - } - - final String routingField = pluginSetting.getStringOrDefault(ROUTING_FIELD, null); - final String routing = pluginSetting.getStringOrDefault(ROUTING, null); - if (routingField != null) { - LOG.warn("routing_field is deprecated in favor of routing, and support for routing_field will be removed in a future major version release."); - builder = builder.withRoutingField(routingField); - } else if (routing != null) { - builder = builder.withRouting(routing); - } - - final String pipeline = pluginSetting.getStringOrDefault(PIPELINE, null); - if (pipeline != null) { - builder = builder.withPipeline(pipeline); + List actionList = openSearchSinkConfig.getActions(); + if (actionList != null) { + builder = builder.withActions(actionList, expressionEvaluator); + } else { + builder = builder.withAction(openSearchSinkConfig.getAction(), expressionEvaluator); } - final String ismPolicyFile = pluginSetting.getStringOrDefault(ISM_POLICY_FILE, null); - builder = builder.withIsmPolicyFile(ismPolicyFile); - - List> actionsList = pluginSetting.getTypedListOfMaps(ACTIONS, String.class, Object.class); - - if (actionsList != null) { - builder.withActions(actionsList, expressionEvaluator); + AwsAuthenticationConfiguration awsAuthenticationConfiguration = openSearchSinkConfig.getAwsAuthenticationOptions(); + if (awsAuthenticationConfiguration != null) { + builder = builder.withServerless(awsAuthenticationConfiguration.isServerlessCollection()); } else { - builder.withAction(pluginSetting.getStringOrDefault(ACTION, OpenSearchBulkActions.INDEX.toString()), expressionEvaluator); + builder = builder.withServerless(false); } if ((builder.templateFile != null && builder.templateFile.startsWith(S3_PREFIX)) - || (builder.ismPolicyFile.isPresent() && builder.ismPolicyFile.get().startsWith(S3_PREFIX))) { - builder.withS3AwsRegion(pluginSetting.getStringOrDefault(S3_AWS_REGION, DEFAULT_AWS_REGION)); - builder.withS3AWSStsRoleArn(pluginSetting.getStringOrDefault(S3_AWS_STS_ROLE_ARN, null)); - builder.withS3AWSStsExternalId(pluginSetting.getStringOrDefault(S3_AWS_STS_EXTERNAL_ID, null)); + || (builder.ismPolicyFile.isPresent() && builder.ismPolicyFile.get().startsWith(S3_PREFIX))) { + builder.withS3AwsRegion(awsAuthenticationConfiguration.getAwsRegion().toString()); + builder.withS3AWSStsRoleArn(awsAuthenticationConfiguration.getAwsStsRoleArn()); + builder.withS3AWSStsExternalId(awsAuthenticationConfiguration.getAwsStsExternalId()); final S3ClientProvider clientProvider = new S3ClientProvider( - builder.s3AwsRegion, builder.s3AwsStsRoleArn, builder.s3AwsStsExternalId); + builder.s3AwsRegion, builder.s3AwsStsRoleArn, builder.s3AwsStsExternalId); builder.withS3Client(clientProvider.buildS3Client()); } - Map awsOption = pluginSetting.getTypedMap(AWS_OPTION, String.class, Object.class); - if (awsOption != null && !awsOption.isEmpty()) { - builder.withServerless(OBJECT_MAPPER.convertValue( - awsOption.getOrDefault(SERVERLESS, false), Boolean.class)); - } else { - builder.withServerless(false); - } - - final String documentRootKey = pluginSetting.getStringOrDefault(DOCUMENT_ROOT_KEY, null); - builder.withDocumentRootKey(documentRootKey); + return builder.build(); + } - final String distributionVersion = pluginSetting.getStringOrDefault(DISTRIBUTION_VERSION, - DistributionVersion.DEFAULT.getVersion()); - builder.withDistributionVersion(distributionVersion); + public static IndexConfiguration readIndexConfig(final PluginSetting pluginSetting) { + return readIndexConfig(pluginSetting, null); + } + public static IndexConfiguration readIndexConfig(final PluginSetting pluginSetting, final ExpressionEvaluator expressionEvaluator) { + IndexConfiguration.Builder builder = new IndexConfiguration.Builder(); return builder.build(); } +// public static IndexConfiguration readIndexConfig(final PluginSetting pluginSetting, final ExpressionEvaluator expressionEvaluator) { +// IndexConfiguration.Builder builder = new IndexConfiguration.Builder(); +// final String indexAlias = pluginSetting.getStringOrDefault(INDEX_ALIAS, null); +// if (indexAlias != null) { +// builder = builder.withIndexAlias(indexAlias); +// } +// final String indexType = pluginSetting.getStringOrDefault(INDEX_TYPE, null); +// if(indexType != null) { +// builder = builder.withIndexType(indexType); +// } +// final String templateType = pluginSetting.getStringOrDefault(TEMPLATE_TYPE, TemplateType.V1.getTypeName()); +// if(templateType != null) { +// builder = builder.withTemplateType(templateType); +// } +// final String templateFile = pluginSetting.getStringOrDefault(TEMPLATE_FILE, null); +// if (templateFile != null) { +// builder = builder.withTemplateFile(templateFile); +// } +// +// final String templateContent = pluginSetting.getStringOrDefault(TEMPLATE_CONTENT, null); +// if (templateContent != null) { +// builder = builder.withTemplateContent(templateContent); +// } +// +// if (templateContent != null && templateFile != null) { +// LOG.warn("Both template_content and template_file are configured. Only template_content will be used"); +// } +// +// builder = builder.withNumShards(pluginSetting.getIntegerOrDefault(NUM_SHARDS, 0)); +// builder = builder.withNumReplicas(pluginSetting.getIntegerOrDefault(NUM_REPLICAS, 0)); +// final Long batchSize = pluginSetting.getLongOrDefault(BULK_SIZE, DEFAULT_BULK_SIZE); +// builder = builder.withBulkSize(batchSize); +// final boolean estimateBulkSizeUsingCompression = +// pluginSetting.getBooleanOrDefault(ESTIMATE_BULK_SIZE_USING_COMPRESSION, DEFAULT_ESTIMATE_BULK_SIZE_USING_COMPRESSION); +// builder = builder.withEstimateBulkSizeUsingCompression(estimateBulkSizeUsingCompression); +// +// final int maxLocalCompressionsForEstimation = +// pluginSetting.getIntegerOrDefault(MAX_LOCAL_COMPRESSIONS_FOR_ESTIMATION, DEFAULT_MAX_LOCAL_COMPRESSIONS_FOR_ESTIMATION); +// builder = builder.withMaxLocalCompressionsForEstimation(maxLocalCompressionsForEstimation); +// +// final long flushTimeout = pluginSetting.getLongOrDefault(FLUSH_TIMEOUT, DEFAULT_FLUSH_TIMEOUT); +// builder = builder.withFlushTimeout(flushTimeout); +// +// final String versionExpression = pluginSetting.getStringOrDefault(DOCUMENT_VERSION_EXPRESSION, null); +// final String versionType = pluginSetting.getStringOrDefault(DOCUMENT_VERSION_TYPE, null); +// final boolean normalizeIndex = pluginSetting.getBooleanOrDefault(NORMALIZE_INDEX, false); +// builder = builder.withNormalizeIndex(normalizeIndex); +// +// builder = builder.withVersionExpression(versionExpression); +// if (versionExpression != null && (!expressionEvaluator.isValidFormatExpression(versionExpression))) { +// throw new InvalidPluginConfigurationException( +// String.format("document_version \"%s\" is not a valid format expression.", versionExpression)); +// } +// +// builder = builder.withVersionType(versionType); +// +// +// final String documentIdField = pluginSetting.getStringOrDefault(DOCUMENT_ID_FIELD, null); +// final String documentId = pluginSetting.getStringOrDefault(DOCUMENT_ID, null); +// if (Objects.nonNull(documentIdField) && Objects.nonNull(documentId)) { +// throw new InvalidPluginConfigurationException("Both document_id_field and document_id cannot be used at the same time. It is preferred to only use document_id as document_id_field is deprecated."); +// } +// +// if (documentIdField != null) { +// LOG.warn("document_id_field is deprecated in favor of document_id, and support for document_id_field will be removed in a future major version release."); +// builder = builder.withDocumentIdField(documentIdField); +// } else if (documentId != null) { +// builder = builder.withDocumentId(documentId); +// } +// +// final String routingField = pluginSetting.getStringOrDefault(ROUTING_FIELD, null); +// final String routing = pluginSetting.getStringOrDefault(ROUTING, null); +// if (routingField != null) { +// LOG.warn("routing_field is deprecated in favor of routing, and support for routing_field will be removed in a future major version release."); +// builder = builder.withRoutingField(routingField); +// } else if (routing != null) { +// builder = builder.withRouting(routing); +// } +// +// final String pipeline = pluginSetting.getStringOrDefault(PIPELINE, null); +// if (pipeline != null) { +// builder = builder.withPipeline(pipeline); +// } +// +// final String ismPolicyFile = pluginSetting.getStringOrDefault(ISM_POLICY_FILE, null); +// builder = builder.withIsmPolicyFile(ismPolicyFile); +// +// List> actionsList = pluginSetting.getTypedListOfMaps(ACTIONS, String.class, Object.class); +// +// if (actionsList != null) { +// builder.withActions(actionsList, expressionEvaluator); +// } else { +// builder.withAction(pluginSetting.getStringOrDefault(ACTION, OpenSearchBulkActions.INDEX.toString()), expressionEvaluator); +// } +// +// if ((builder.templateFile != null && builder.templateFile.startsWith(S3_PREFIX)) +// || (builder.ismPolicyFile.isPresent() && builder.ismPolicyFile.get().startsWith(S3_PREFIX))) { +// builder.withS3AwsRegion(pluginSetting.getStringOrDefault(S3_AWS_REGION, DEFAULT_AWS_REGION)); +// builder.withS3AWSStsRoleArn(pluginSetting.getStringOrDefault(S3_AWS_STS_ROLE_ARN, null)); +// builder.withS3AWSStsExternalId(pluginSetting.getStringOrDefault(S3_AWS_STS_EXTERNAL_ID, null)); +// +// final S3ClientProvider clientProvider = new S3ClientProvider( +// builder.s3AwsRegion, builder.s3AwsStsRoleArn, builder.s3AwsStsExternalId); +// builder.withS3Client(clientProvider.buildS3Client()); +// } +// +// Map awsOption = pluginSetting.getTypedMap(AWS_OPTION, String.class, Object.class); +// if (awsOption != null && !awsOption.isEmpty()) { +// builder.withServerless(OBJECT_MAPPER.convertValue( +// awsOption.getOrDefault(SERVERLESS, false), Boolean.class)); +// } else { +// builder.withServerless(false); +// } +// +// final String documentRootKey = pluginSetting.getStringOrDefault(DOCUMENT_ROOT_KEY, null); +// builder.withDocumentRootKey(documentRootKey); +// +// final String distributionVersion = pluginSetting.getStringOrDefault(DISTRIBUTION_VERSION, +// DistributionVersion.DEFAULT.getVersion()); +// builder.withDistributionVersion(distributionVersion); +// +// return builder.build(); +// } + public IndexType getIndexType() { return indexType; } @@ -345,10 +439,6 @@ public String getRouting() { return routing; } - public String getPipeline() { - return pipeline; - } - public long getBulkSize() { return bulkSize; } @@ -373,7 +463,7 @@ public String getAction() { return action; } - public List> getActions() { + public List getActions() { return actions; } @@ -485,7 +575,7 @@ public static class Builder { private long flushTimeout = DEFAULT_FLUSH_TIMEOUT; private Optional ismPolicyFile; private String action; - private List> actions; + private List actions; private String s3AwsRegion; private String s3AwsStsRoleArn; private String s3AwsStsExternalId; @@ -552,11 +642,6 @@ public Builder withRouting(final String routing) { return this; } - public Builder withPipeline(final String pipeline) { - this.pipeline = pipeline; - return this; - } - public Builder withBulkSize(final long bulkSize) { this.bulkSize = bulkSize; return this; @@ -599,9 +684,9 @@ public Builder withAction(final String action, final ExpressionEvaluator express return this; } - public Builder withActions(final List> actions, final ExpressionEvaluator expressionEvaluator) { - for (final Map actionMap: actions) { - String action = (String)actionMap.get("type"); + public Builder withActions(final List actions, final ExpressionEvaluator expressionEvaluator) { + for (final ActionConfiguration actionConfig: actions) { + String action = actionConfig.getType(); if (action != null) { checkArgument((EnumUtils.isValidEnumIgnoreCase(OpenSearchBulkActions.class, action) || (action.contains("${") && expressionEvaluator.isValidFormatExpression(action))), "action \"" + action + "\". action must be one of the following: " + Arrays.stream(OpenSearchBulkActions.values()).collect(Collectors.toList())); diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/TemplateType.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/TemplateType.java index 0c48d8ef41..7b8020a19a 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/TemplateType.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/TemplateType.java @@ -46,7 +46,7 @@ public TemplateStrategy createTemplateStrategy(final IndexTemplateAPIWrapper ind return factoryFunction.apply(indexTemplateAPIWrapper); } - String getTypeName() { + public String getTypeName() { return name; } } From acbfb7b35fc2f49973a07cf0fc25c82839744fe7 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Tue, 10 Dec 2024 18:02:43 -0800 Subject: [PATCH 03/62] rebuild readRetryConfig. Remove most instances of PluginSetting from OpenSearchSink. Remove commented out code Signed-off-by: Maxwell Brown --- .../sink/opensearch/BulkRetryStrategy.java | 10 +- .../opensearch/ConnectionConfiguration.java | 107 +------------- .../opensearch/OpenSearchClientRefresher.java | 8 +- .../sink/opensearch/OpenSearchSink.java | 43 +++--- .../sink/opensearch/OpenSearchSinkConfig.java | 21 +++ .../OpenSearchSinkConfiguration.java | 16 +-- .../sink/opensearch/RetryConfiguration.java | 48 +++---- .../configuration/DlqConfiguration.java | 13 ++ .../opensearch/index/IndexConfiguration.java | 130 ------------------ 9 files changed, 88 insertions(+), 308 deletions(-) create mode 100644 data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/DlqConfiguration.java diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategy.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategy.java index aca748b3ef..9deea290e6 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategy.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategy.java @@ -15,7 +15,6 @@ import org.opensearch.client.opensearch.core.BulkResponse; import org.opensearch.client.opensearch.core.bulk.BulkResponseItem; import org.opensearch.dataprepper.metrics.PluginMetrics; -import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.AccumulatingBulkRequest; import org.opensearch.dataprepper.plugins.sink.opensearch.dlq.FailedBulkOperation; import org.opensearch.rest.RestStatus; @@ -150,15 +149,16 @@ public BulkRetryStrategy(final RequestFunction bulkRequestSupplier, - final PluginSetting pluginSetting) { + final String pipelineName, + final String pluginName) { this.requestFunction = requestFunction; this.logFailure = logFailure; this.pluginMetrics = pluginMetrics; this.bulkRequestSupplier = bulkRequestSupplier; this.maxRetries = maxRetries; - this.pipelineName = pluginSetting.getPipelineName(); - this.pluginId = pluginSetting.getName(); - this.pluginName = pluginSetting.getName(); + this.pipelineName = pipelineName; + this.pluginId = pluginName; + this.pluginName = pluginName; this.objectMapper = new ObjectMapper(); sentDocumentsCounter = pluginMetrics.counter(DOCUMENTS_SUCCESS); diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java index e76d9189a0..511fc87a29 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java @@ -5,7 +5,6 @@ package org.opensearch.dataprepper.plugins.sink.opensearch; -import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.http.HttpHost; import org.apache.http.HttpRequestInterceptor; import org.apache.http.auth.AuthScope; @@ -65,33 +64,15 @@ import static com.google.common.base.Preconditions.checkNotNull; public class ConnectionConfiguration { - static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final Logger LOG = LoggerFactory.getLogger(OpenSearchSink.class); private static final String AWS_IAM_ROLE = "role"; private static final String AWS_IAM = "iam"; private static final String AOS_SERVICE_NAME = "es"; private static final String AOSS_SERVICE_NAME = "aoss"; - private static final String DEFAULT_AWS_REGION = "us-east-1"; public static final String HOSTS = "hosts"; - public static final String USERNAME = "username"; - public static final String PASSWORD = "password"; - public static final String SOCKET_TIMEOUT = "socket_timeout"; - public static final String CONNECT_TIMEOUT = "connect_timeout"; - public static final String CERT_PATH = "cert"; - public static final String INSECURE = "insecure"; - public static final String AWS_OPTION = "aws"; public static final String AWS_SIGV4 = "aws_sigv4"; - public static final String AWS_REGION = "aws_region"; - public static final String AWS_STS_ROLE_ARN = "aws_sts_role_arn"; - public static final String AWS_STS_EXTERNAL_ID = "aws_sts_external_id"; - public static final String AWS_STS_HEADER_OVERRIDES = "aws_sts_header_overrides"; - public static final String PROXY = "proxy"; public static final String SERVERLESS = "serverless"; - public static final String SERVERLESS_OPTIONS = "serverless_options"; - public static final String COLLECTION_NAME = "collection_name"; - public static final String NETWORK_POLICY_NAME = "network_policy_name"; - public static final String VPCE_ID = "vpce_id"; public static final String REQUEST_COMPRESSION_ENABLED = "enable_request_compression"; /** @@ -273,93 +254,7 @@ public static ConnectionConfiguration readConnectionConfiguration(final PluginSe ConnectionConfiguration.Builder builder = new ConnectionConfiguration.Builder(hosts); return builder.build(); } -// -// public static ConnectionConfiguration readConnectionConfiguration(final PluginSetting pluginSetting){ -// @SuppressWarnings("unchecked") -// final List hosts = (List) pluginSetting.getAttributeFromSettings(HOSTS); -// ConnectionConfiguration.Builder builder = new ConnectionConfiguration.Builder(hosts); -// final String username = (String) pluginSetting.getAttributeFromSettings(USERNAME); -// final String password = (String) pluginSetting.getAttributeFromSettings(PASSWORD); -// builder.withPipelineName(pluginSetting.getPipelineName()); -// final AuthConfig authConfig = AuthConfig.readAuthConfig(pluginSetting); -// if (authConfig != null) { -// if (username != null || password != null) { -// throw new IllegalStateException("Deprecated username and password should not be set " + -// "when authentication is configured."); -// } -// builder = builder.withAuthConfig(authConfig); -// } else { -// if (username != null) { -// builder = builder.withUsername(username); -// } -// if (password != null) { -// builder = builder.withPassword(password); -// } -// } -// final Integer socketTimeout = pluginSetting.getIntegerOrDefault(SOCKET_TIMEOUT, null); -// if (socketTimeout != null) { -// builder = builder.withSocketTimeout(socketTimeout); -// } -// final Integer connectTimeout = pluginSetting.getIntegerOrDefault(CONNECT_TIMEOUT, null); -// if (connectTimeout != null) { -// builder = builder.withConnectTimeout(connectTimeout); -// } -// -// Map awsOption = pluginSetting.getTypedMap(AWS_OPTION, String.class, Object.class); -// boolean awsOptionUsed = false; -// builder.withAwsSigv4(false); -// if (awsOption != null && !awsOption.isEmpty()) { -// awsOptionUsed = true; -// builder.withAwsSigv4(true); -// builder.withAwsRegion((String)(awsOption.getOrDefault(AWS_REGION.substring(4), DEFAULT_AWS_REGION))); -// builder.withAWSStsRoleArn((String)(awsOption.getOrDefault(AWS_STS_ROLE_ARN.substring(4), null))); -// builder.withAWSStsExternalId((String)(awsOption.getOrDefault(AWS_STS_EXTERNAL_ID.substring(4), null))); -// builder.withAwsStsHeaderOverrides((Map)awsOption.get(AWS_STS_HEADER_OVERRIDES.substring(4))); -// builder.withServerless(OBJECT_MAPPER.convertValue( -// awsOption.getOrDefault(SERVERLESS, false), Boolean.class)); -// -// Map serverlessOptions = (Map) awsOption.get(SERVERLESS_OPTIONS); -// if (serverlessOptions != null && !serverlessOptions.isEmpty()) { -// builder.withServerlessNetworkPolicyName((String)(serverlessOptions.getOrDefault(NETWORK_POLICY_NAME, null))); -// builder.withServerlessCollectionName((String)(serverlessOptions.getOrDefault(COLLECTION_NAME, null))); -// builder.withServerlessVpceId((String)(serverlessOptions.getOrDefault(VPCE_ID, null))); -// } -// } else { -// builder.withServerless(false); -// } -// boolean awsSigv4 = pluginSetting.getBooleanOrDefault(AWS_SIGV4, false); -// final String awsOptionConflictMessage = String.format("%s option cannot be used along with %s option", AWS_SIGV4, AWS_OPTION); -// if (awsSigv4) { -// if (awsOptionUsed) { -// throw new RuntimeException(awsOptionConflictMessage); -// } -// builder.withAwsSigv4(true); -// builder.withAwsRegion(pluginSetting.getStringOrDefault(AWS_REGION, DEFAULT_AWS_REGION)); -// builder.withAWSStsRoleArn(pluginSetting.getStringOrDefault(AWS_STS_ROLE_ARN, null)); -// builder.withAWSStsExternalId(pluginSetting.getStringOrDefault(AWS_STS_EXTERNAL_ID, null)); -// builder.withAwsStsHeaderOverrides(pluginSetting.getTypedMap(AWS_STS_HEADER_OVERRIDES, String.class, String.class)); -// } -// -// final String certPath = pluginSetting.getStringOrDefault(CERT_PATH, null); -// final boolean insecure = pluginSetting.getBooleanOrDefault(INSECURE, false); -// if (certPath != null) { -// builder = builder.withCert(certPath); -// } else { -// //We will set insecure flag only if certPath is null -// builder = builder.withInsecure(insecure); -// } -// final String proxy = pluginSetting.getStringOrDefault(PROXY, null); -// builder = builder.withProxy(proxy); -// -// final String distributionVersionName = pluginSetting.getStringOrDefault(DISTRIBUTION_VERSION, -// DistributionVersion.DEFAULT.getVersion()); -// final DistributionVersion distributionVersion = DistributionVersion.fromTypeName(distributionVersionName); -// final boolean requestCompressionEnabled = pluginSetting.getBooleanOrDefault( -// REQUEST_COMPRESSION_ENABLED, !DistributionVersion.ES6.equals(distributionVersion)); -// builder = builder.withRequestCompressionEnabled(requestCompressionEnabled); -// -// return builder.build(); -// } + public RestHighLevelClient createClient(AwsCredentialsSupplier awsCredentialsSupplier) { final HttpHost[] httpHosts = new HttpHost[hosts.size()]; diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresher.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresher.java index b697fb26bf..bee6242641 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresher.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresher.java @@ -54,8 +54,12 @@ public OpenSearchClient get() { } @Override - public void update(PluginSetting pluginSetting) { - final ConnectionConfiguration newConfig = ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + public void update(final PluginSetting pluginSetting) { + return; + } + + public void update(OpenSearchSinkConfig openSearchSinkConfig) { + final ConnectionConfiguration newConfig = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); if (basicAuthChanged(newConfig)) { credentialsChangeCounter.increment(); readWriteLock.writeLock().lock(); diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java index b58e8165c5..09806d57af 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java @@ -29,7 +29,6 @@ import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; import org.opensearch.dataprepper.model.configuration.PipelineDescription; -import org.opensearch.dataprepper.model.configuration.PluginModel; import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.event.exceptions.EventKeyNotFoundException; @@ -37,7 +36,6 @@ import org.opensearch.dataprepper.model.opensearch.OpenSearchBulkActions; import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException; import org.opensearch.dataprepper.model.plugin.PluginConfigObservable; -import org.opensearch.dataprepper.model.plugin.PluginFactory; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.sink.AbstractSink; import org.opensearch.dataprepper.model.sink.Sink; @@ -45,8 +43,8 @@ import org.opensearch.dataprepper.plugins.common.opensearch.ServerlessNetworkPolicyUpdater; import org.opensearch.dataprepper.plugins.common.opensearch.ServerlessNetworkPolicyUpdaterFactory; import org.opensearch.dataprepper.plugins.common.opensearch.ServerlessOptionsFactory; -import org.opensearch.dataprepper.plugins.dlq.DlqProvider; import org.opensearch.dataprepper.plugins.dlq.DlqWriter; +import org.opensearch.dataprepper.plugins.dlq.s3.S3DlqProvider; import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.AccumulatingBulkRequest; import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.BulkApiWrapper; import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.BulkApiWrapperFactory; @@ -55,6 +53,7 @@ import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.JavaClientAccumulatingUncompressedBulkRequest; import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.SerializedJson; import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.ActionConfiguration; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.DlqConfiguration; import org.opensearch.dataprepper.plugins.sink.opensearch.dlq.FailedBulkOperation; import org.opensearch.dataprepper.plugins.sink.opensearch.dlq.FailedBulkOperationConverter; import org.opensearch.dataprepper.plugins.sink.opensearch.dlq.FailedDlqData; @@ -78,7 +77,6 @@ import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.StringJoiner; @@ -96,6 +94,7 @@ public class OpenSearchSink extends AbstractSink> { public static final String BULKREQUEST_SIZE_BYTES = "bulkRequestSizeBytes"; public static final String DYNAMIC_INDEX_DROPPED_EVENTS = "dynamicIndexDroppedEvents"; public static final String INVALID_VERSION_EXPRESSION_DROPPED_EVENTS = "dynamicDocumentVersionDroppedEvents"; + private static final String PLUGIN_NAME = "opensearch"; private static final Logger LOG = LoggerFactory.getLogger(OpenSearchSink.class); private static final int INITIALIZE_RETRY_WAIT_TIME_MS = 5000; @@ -136,20 +135,18 @@ public class OpenSearchSink extends AbstractSink> { private OpenSearchClientRefresher openSearchClientRefresher; private ObjectMapper objectMapper; private volatile boolean initialized; - private PluginSetting pluginSetting; private final SinkContext sinkContext; private final ExpressionEvaluator expressionEvaluator; private FailedBulkOperationConverter failedBulkOperationConverter; - private DlqProvider dlqProvider; + private S3DlqProvider s3DlqProvider; private final ConcurrentHashMap> bulkRequestMap; private final ConcurrentHashMap lastFlushTimeMap; private final PluginConfigObservable pluginConfigObservable; @DataPrepperPluginConstructor public OpenSearchSink(final PluginSetting pluginSetting, - final PluginFactory pluginFactory, final SinkContext sinkContext, final ExpressionEvaluator expressionEvaluator, final AwsCredentialsSupplier awsCredentialsSupplier, @@ -182,21 +179,18 @@ public OpenSearchSink(final PluginSetting pluginSetting, this.versionType = openSearchSinkConfig.getIndexConfiguration().getVersionType(); this.versionExpression = openSearchSinkConfig.getIndexConfiguration().getVersionExpression(); this.indexManagerFactory = new IndexManagerFactory(new ClusterSettingsParser()); - this.failedBulkOperationConverter = new FailedBulkOperationConverter(pluginSetting.getPipelineName(), pluginSetting.getName(), - pluginSetting.getName()); + this.failedBulkOperationConverter = new FailedBulkOperationConverter(pipeline, PLUGIN_NAME, + PLUGIN_NAME); this.initialized = false; this.lock = new ReentrantLock(true); - this.pluginSetting = pluginSetting; this.bulkRequestMap = new ConcurrentHashMap<>(); this.lastFlushTimeMap = new ConcurrentHashMap<>(); this.pluginConfigObservable = pluginConfigObservable; this.objectMapper = new ObjectMapper(); - final Optional dlqConfig = openSearchSinkConfig.getRetryConfiguration().getDlq(); + final Optional dlqConfig = openSearchSinkConfig.getRetryConfiguration().getDlq(); if (dlqConfig.isPresent()) { - final PluginSetting dlqPluginSetting = new PluginSetting(dlqConfig.get().getPluginName(), dlqConfig.get().getPluginSettings()); - dlqPluginSetting.setPipelineName(pluginSetting.getPipelineName()); - dlqProvider = pluginFactory.loadPlugin(DlqProvider.class, dlqPluginSetting); + s3DlqProvider = new S3DlqProvider(dlqConfig.get().getS3DlqWriterConfig()); } } @@ -237,7 +231,7 @@ private void doInitializeInternal() throws IOException { openSearchClientRefresher = new OpenSearchClientRefresher( pluginMetrics, connectionConfiguration, clientFunction); pluginConfigObservable.addPluginConfigObserver( - newPluginSetting -> openSearchClientRefresher.update((PluginSetting) newPluginSetting)); + newOpenSearchSinkConfig -> openSearchClientRefresher.update((OpenSearchSinkConfig) newOpenSearchSinkConfig)); configuredIndexAlias = openSearchSinkConfig.getIndexConfiguration().getIndexAlias(); final IndexTemplateAPIWrapper indexTemplateAPIWrapper = IndexTemplateAPIWrapperFactory.getWrapper( openSearchSinkConfig.getIndexConfiguration(), openSearchClient); @@ -248,10 +242,10 @@ private void doInitializeInternal() throws IOException { final String dlqFile = openSearchSinkConfig.getRetryConfiguration().getDlqFile(); if (dlqFile != null) { dlqFileWriter = Files.newBufferedWriter(Paths.get(dlqFile), StandardOpenOption.CREATE, StandardOpenOption.APPEND); - } else if (dlqProvider != null) { - Optional potentialDlq = dlqProvider.getDlqWriter(new StringJoiner(MetricNames.DELIMITER) - .add(pluginSetting.getPipelineName()) - .add(pluginSetting.getName()).toString()); + } else if (s3DlqProvider != null) { + Optional potentialDlq = s3DlqProvider.getDlqWriter(new StringJoiner(MetricNames.DELIMITER) + .add(pipeline) + .add(PLUGIN_NAME).toString()); dlqWriter = potentialDlq.isPresent() ? potentialDlq.get() : null; } @@ -282,7 +276,8 @@ private void doInitializeInternal() throws IOException { pluginMetrics, maxRetries, bulkRequestSupplier, - pluginSetting); + pipeline, + PLUGIN_NAME); this.initialized = true; LOG.info("Initialized OpenSearch sink"); @@ -568,7 +563,7 @@ private void logFailureForDlqObjects(final List dlqObjects, final Thr }); } else if (dlqWriter != null) { try { - dlqWriter.write(dlqObjects, pluginSetting.getPipelineName(), pluginSetting.getName()); + dlqWriter.write(dlqObjects, pipeline, PLUGIN_NAME); dlqObjects.forEach((dlqObject) -> { dlqObject.releaseEventHandle(true); }); @@ -648,9 +643,9 @@ private DlqObject createDlqObjectFromEvent(final Event event, .withIndex(index) .withMessage(message) .build()) - .withPluginName(pluginSetting.getName()) - .withPipelineName(pluginSetting.getPipelineName()) - .withPluginId(pluginSetting.getName()) + .withPluginName(PLUGIN_NAME) + .withPipelineName(pipeline) + .withPluginId(PLUGIN_NAME) .build(); } diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfig.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfig.java index c7b9eb4ffd..ba02849f65 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfig.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfig.java @@ -5,6 +5,7 @@ import lombok.Getter; import org.opensearch.dataprepper.model.opensearch.OpenSearchBulkActions; import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.ActionConfiguration; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.DlqConfiguration; import org.opensearch.dataprepper.plugins.sink.opensearch.index.TemplateType; import org.opensearch.dataprepper.plugins.source.opensearch.configuration.AwsAuthenticationConfiguration; @@ -153,12 +154,32 @@ public Long getFlushTimeout(Long defaultFlushTimeout) { @JsonProperty("document_root_key") private String documentRootKey = null; + @Getter + @JsonProperty("dlq_file") + private String dlqFile = null; + + @Getter + @JsonProperty("max_retries") + private Integer maxRetries = null; + + @Getter + @JsonProperty("dlq") + private DlqConfiguration dlq; + public void validateConfig() { isActionValid(); + isDlqValid(); } + void isDlqValid() { + if (dlq != null) { + if (dlqFile!= null) { + throw new IllegalArgumentException("dlq_file option cannot be used along with dlq option"); + } + } + } void isActionValid() { if (action.equals("index") || action.equals("create") || action.equals("update") || action.equals("upsert") || action.equals("delete")) { diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfiguration.java index b3bfe801ac..3c563ea5c0 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfiguration.java @@ -5,9 +5,8 @@ package org.opensearch.dataprepper.plugins.sink.opensearch; -import org.opensearch.dataprepper.model.configuration.PluginSetting; -import org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration; import org.opensearch.dataprepper.expression.ExpressionEvaluator; +import org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration; import static com.google.common.base.Preconditions.checkNotNull; @@ -40,19 +39,6 @@ private OpenSearchSinkConfiguration( this.retryConfiguration = retryConfiguration; } -// public static OpenSearchSinkConfiguration readESConfig(final PluginSetting pluginSetting) { -// return readESConfig(pluginSetting, null); -// } -// -// public static OpenSearchSinkConfiguration readESConfig(final PluginSetting pluginSetting, final ExpressionEvaluator expressionEvaluator) { -// final ConnectionConfiguration connectionConfiguration = -// ConnectionConfiguration.readConnectionConfiguration(pluginSetting); -// final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting, expressionEvaluator); -// final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(pluginSetting); -// -// return new OpenSearchSinkConfiguration(connectionConfiguration, indexConfiguration, retryConfiguration); -// } - public static OpenSearchSinkConfiguration readOSConfig(final OpenSearchSinkConfig openSearchSinkConfig, final ExpressionEvaluator expressionEvaluator) { openSearchSinkConfig.validateConfig(); final ConnectionConfiguration connectionConfiguration = diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfiguration.java index fa397e9271..3bc2c7f990 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfiguration.java @@ -7,6 +7,7 @@ import org.opensearch.dataprepper.model.configuration.PluginModel; import org.opensearch.dataprepper.model.configuration.PluginSetting; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.DlqConfiguration; import java.util.LinkedHashMap; import java.util.Map; @@ -21,13 +22,13 @@ public class RetryConfiguration { private final String dlqFile; private final int maxRetries; - private final PluginModel dlq; + private final DlqConfiguration dlq; public String getDlqFile() { return dlqFile; } - public Optional getDlq() { + public Optional getDlq() { return Optional.ofNullable(dlq); } @@ -42,7 +43,7 @@ public static class Builder { private String dlqFile; private int maxRetries = Integer.MAX_VALUE; - private PluginModel dlq; + private DlqConfiguration dlq; public Builder withDlqFile(final String dlqFile) { checkNotNull(dlqFile, "dlqFile cannot be null."); @@ -56,7 +57,7 @@ public Builder withMaxRetries(final Integer maxRetries) { return this; } - public Builder withDlq(final PluginModel dlq) { + public Builder withDlq(final DlqConfiguration dlq) { checkNotNull(dlq, "dlq cannot be null"); this.dlq = dlq; return this; @@ -72,30 +73,25 @@ private RetryConfiguration(final Builder builder) { this.dlq = builder.dlq; } + public static RetryConfiguration readRetryConfig(final OpenSearchSinkConfig openSearchSinkConfig) { + RetryConfiguration.Builder builder = new RetryConfiguration.Builder(); + final String dlqFile = openSearchSinkConfig.getDlqFile(); + if (dlqFile != null) { + builder = builder.withDlqFile(dlqFile); + } + final Integer maxRetries = openSearchSinkConfig.getMaxRetries(); + if (maxRetries != null) { + builder = builder.withMaxRetries(maxRetries); + } + final DlqConfiguration dlqConfiguration = openSearchSinkConfig.getDlq(); + if (dlqConfiguration != null) { + builder = builder.withDlq(dlqConfiguration); + } + return builder.build(); + } + public static RetryConfiguration readRetryConfig(final PluginSetting pluginSetting) { RetryConfiguration.Builder builder = new RetryConfiguration.Builder(); - final String dlqFile = (String) pluginSetting.getAttributeFromSettings(DLQ_FILE); - if (dlqFile != null) { - builder = builder.withDlqFile(dlqFile); - } - final Integer maxRetries = pluginSetting.getIntegerOrDefault(MAX_RETRIES, null); - if (maxRetries != null) { - builder = builder.withMaxRetries(maxRetries); - } - final LinkedHashMap> dlq = (LinkedHashMap) pluginSetting.getAttributeFromSettings(DLQ); - if (dlq != null) { - if (dlqFile != null) { - final String dlqOptionConflictMessage = String.format("%s option cannot be used along with %s option", DLQ_FILE, DLQ); - throw new RuntimeException(dlqOptionConflictMessage); - } - if (dlq.size() != 1) { - throw new RuntimeException("dlq option must declare exactly one dlq configuration"); - } - final Map.Entry> entry = dlq.entrySet().stream() - .findFirst() - .get(); - builder = builder.withDlq(new PluginModel(entry.getKey(), entry.getValue())); - } return builder.build(); } } diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/DlqConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/DlqConfiguration.java new file mode 100644 index 0000000000..ea82e33cd3 --- /dev/null +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/DlqConfiguration.java @@ -0,0 +1,13 @@ +package org.opensearch.dataprepper.plugins.sink.opensearch.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.Valid; +import lombok.Getter; +import org.opensearch.dataprepper.plugins.dlq.s3.S3DlqWriterConfig; + +public class DlqConfiguration { + @Getter + @Valid + @JsonProperty("s3") + private S3DlqWriterConfig s3DlqWriterConfig; +} diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java index 4de7c844d7..749d9361d1 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java @@ -25,7 +25,6 @@ import software.amazon.awssdk.arns.Arn; import software.amazon.awssdk.services.s3.S3Client; -import javax.swing.*; import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -34,7 +33,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; @@ -69,16 +67,11 @@ public class IndexConfiguration { public static final long DEFAULT_FLUSH_TIMEOUT = 60_000L; public static final String ACTION = "action"; public static final String ACTIONS = "actions"; - public static final String S3_AWS_REGION = "s3_aws_region"; - public static final String S3_AWS_STS_ROLE_ARN = "s3_aws_sts_role_arn"; - public static final String S3_AWS_STS_EXTERNAL_ID = "s3_aws_sts_external_id"; public static final String SERVERLESS = "serverless"; public static final String DISTRIBUTION_VERSION = "distribution_version"; public static final String AWS_OPTION = "aws"; public static final String DOCUMENT_ROOT_KEY = "document_root_key"; public static final String DOCUMENT_VERSION_EXPRESSION = "document_version"; - public static final String DOCUMENT_VERSION_TYPE = "document_version_type"; - public static final String NORMALIZE_INDEX = "normalize_index"; private IndexType indexType; private TemplateType templateType; @@ -285,129 +278,6 @@ public static IndexConfiguration readIndexConfig(final PluginSetting pluginSetti return builder.build(); } -// public static IndexConfiguration readIndexConfig(final PluginSetting pluginSetting, final ExpressionEvaluator expressionEvaluator) { -// IndexConfiguration.Builder builder = new IndexConfiguration.Builder(); -// final String indexAlias = pluginSetting.getStringOrDefault(INDEX_ALIAS, null); -// if (indexAlias != null) { -// builder = builder.withIndexAlias(indexAlias); -// } -// final String indexType = pluginSetting.getStringOrDefault(INDEX_TYPE, null); -// if(indexType != null) { -// builder = builder.withIndexType(indexType); -// } -// final String templateType = pluginSetting.getStringOrDefault(TEMPLATE_TYPE, TemplateType.V1.getTypeName()); -// if(templateType != null) { -// builder = builder.withTemplateType(templateType); -// } -// final String templateFile = pluginSetting.getStringOrDefault(TEMPLATE_FILE, null); -// if (templateFile != null) { -// builder = builder.withTemplateFile(templateFile); -// } -// -// final String templateContent = pluginSetting.getStringOrDefault(TEMPLATE_CONTENT, null); -// if (templateContent != null) { -// builder = builder.withTemplateContent(templateContent); -// } -// -// if (templateContent != null && templateFile != null) { -// LOG.warn("Both template_content and template_file are configured. Only template_content will be used"); -// } -// -// builder = builder.withNumShards(pluginSetting.getIntegerOrDefault(NUM_SHARDS, 0)); -// builder = builder.withNumReplicas(pluginSetting.getIntegerOrDefault(NUM_REPLICAS, 0)); -// final Long batchSize = pluginSetting.getLongOrDefault(BULK_SIZE, DEFAULT_BULK_SIZE); -// builder = builder.withBulkSize(batchSize); -// final boolean estimateBulkSizeUsingCompression = -// pluginSetting.getBooleanOrDefault(ESTIMATE_BULK_SIZE_USING_COMPRESSION, DEFAULT_ESTIMATE_BULK_SIZE_USING_COMPRESSION); -// builder = builder.withEstimateBulkSizeUsingCompression(estimateBulkSizeUsingCompression); -// -// final int maxLocalCompressionsForEstimation = -// pluginSetting.getIntegerOrDefault(MAX_LOCAL_COMPRESSIONS_FOR_ESTIMATION, DEFAULT_MAX_LOCAL_COMPRESSIONS_FOR_ESTIMATION); -// builder = builder.withMaxLocalCompressionsForEstimation(maxLocalCompressionsForEstimation); -// -// final long flushTimeout = pluginSetting.getLongOrDefault(FLUSH_TIMEOUT, DEFAULT_FLUSH_TIMEOUT); -// builder = builder.withFlushTimeout(flushTimeout); -// -// final String versionExpression = pluginSetting.getStringOrDefault(DOCUMENT_VERSION_EXPRESSION, null); -// final String versionType = pluginSetting.getStringOrDefault(DOCUMENT_VERSION_TYPE, null); -// final boolean normalizeIndex = pluginSetting.getBooleanOrDefault(NORMALIZE_INDEX, false); -// builder = builder.withNormalizeIndex(normalizeIndex); -// -// builder = builder.withVersionExpression(versionExpression); -// if (versionExpression != null && (!expressionEvaluator.isValidFormatExpression(versionExpression))) { -// throw new InvalidPluginConfigurationException( -// String.format("document_version \"%s\" is not a valid format expression.", versionExpression)); -// } -// -// builder = builder.withVersionType(versionType); -// -// -// final String documentIdField = pluginSetting.getStringOrDefault(DOCUMENT_ID_FIELD, null); -// final String documentId = pluginSetting.getStringOrDefault(DOCUMENT_ID, null); -// if (Objects.nonNull(documentIdField) && Objects.nonNull(documentId)) { -// throw new InvalidPluginConfigurationException("Both document_id_field and document_id cannot be used at the same time. It is preferred to only use document_id as document_id_field is deprecated."); -// } -// -// if (documentIdField != null) { -// LOG.warn("document_id_field is deprecated in favor of document_id, and support for document_id_field will be removed in a future major version release."); -// builder = builder.withDocumentIdField(documentIdField); -// } else if (documentId != null) { -// builder = builder.withDocumentId(documentId); -// } -// -// final String routingField = pluginSetting.getStringOrDefault(ROUTING_FIELD, null); -// final String routing = pluginSetting.getStringOrDefault(ROUTING, null); -// if (routingField != null) { -// LOG.warn("routing_field is deprecated in favor of routing, and support for routing_field will be removed in a future major version release."); -// builder = builder.withRoutingField(routingField); -// } else if (routing != null) { -// builder = builder.withRouting(routing); -// } -// -// final String pipeline = pluginSetting.getStringOrDefault(PIPELINE, null); -// if (pipeline != null) { -// builder = builder.withPipeline(pipeline); -// } -// -// final String ismPolicyFile = pluginSetting.getStringOrDefault(ISM_POLICY_FILE, null); -// builder = builder.withIsmPolicyFile(ismPolicyFile); -// -// List> actionsList = pluginSetting.getTypedListOfMaps(ACTIONS, String.class, Object.class); -// -// if (actionsList != null) { -// builder.withActions(actionsList, expressionEvaluator); -// } else { -// builder.withAction(pluginSetting.getStringOrDefault(ACTION, OpenSearchBulkActions.INDEX.toString()), expressionEvaluator); -// } -// -// if ((builder.templateFile != null && builder.templateFile.startsWith(S3_PREFIX)) -// || (builder.ismPolicyFile.isPresent() && builder.ismPolicyFile.get().startsWith(S3_PREFIX))) { -// builder.withS3AwsRegion(pluginSetting.getStringOrDefault(S3_AWS_REGION, DEFAULT_AWS_REGION)); -// builder.withS3AWSStsRoleArn(pluginSetting.getStringOrDefault(S3_AWS_STS_ROLE_ARN, null)); -// builder.withS3AWSStsExternalId(pluginSetting.getStringOrDefault(S3_AWS_STS_EXTERNAL_ID, null)); -// -// final S3ClientProvider clientProvider = new S3ClientProvider( -// builder.s3AwsRegion, builder.s3AwsStsRoleArn, builder.s3AwsStsExternalId); -// builder.withS3Client(clientProvider.buildS3Client()); -// } -// -// Map awsOption = pluginSetting.getTypedMap(AWS_OPTION, String.class, Object.class); -// if (awsOption != null && !awsOption.isEmpty()) { -// builder.withServerless(OBJECT_MAPPER.convertValue( -// awsOption.getOrDefault(SERVERLESS, false), Boolean.class)); -// } else { -// builder.withServerless(false); -// } -// -// final String documentRootKey = pluginSetting.getStringOrDefault(DOCUMENT_ROOT_KEY, null); -// builder.withDocumentRootKey(documentRootKey); -// -// final String distributionVersion = pluginSetting.getStringOrDefault(DISTRIBUTION_VERSION, -// DistributionVersion.DEFAULT.getVersion()); -// builder.withDistributionVersion(distributionVersion); -// -// return builder.build(); -// } public IndexType getIndexType() { return indexType; From 2c654800cfe69e556a18309d3abc09e35a58fec9 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Wed, 11 Dec 2024 14:27:38 -0800 Subject: [PATCH 04/62] save point Signed-off-by: Maxwell Brown --- build.gradle | 4 ++++ .../plugins/sink/opensearch/RetryConfiguration.java | 3 --- .../plugins/sink/opensearch/BulkRetryStrategyTests.java | 6 ++++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/build.gradle b/build.gradle index 7f78c2424f..e8fe2d06d3 100644 --- a/build.gradle +++ b/build.gradle @@ -69,9 +69,13 @@ subprojects { } } dependencies { + compileOnly 'org.projectlombok:lombok:1.18.30' + annotationProcessor 'org.projectlombok:lombok:1.18.30' implementation platform('com.fasterxml.jackson:jackson-bom:2.17.2') implementation platform('org.eclipse.jetty:jetty-bom:9.4.53.v20231009') implementation platform('io.micrometer:micrometer-bom:1.10.5') + implementation platform('com.fasterxml.jackson:jackson-bom:2.17.2') + implementation platform('org.eclipse.jetty:jetty-bom:9.4.53.v20231009') implementation libs.guava.core implementation libs.slf4j.api testImplementation testLibs.bundles.junit diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfiguration.java index 3bc2c7f990..ef9ec9ade8 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfiguration.java @@ -5,12 +5,9 @@ package org.opensearch.dataprepper.plugins.sink.opensearch; -import org.opensearch.dataprepper.model.configuration.PluginModel; import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.DlqConfiguration; -import java.util.LinkedHashMap; -import java.util.Map; import java.util.Optional; import static com.google.common.base.Preconditions.checkNotNull; diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategyTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategyTests.java index cc05514502..d30238acb6 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategyTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategyTests.java @@ -124,7 +124,8 @@ public BulkRetryStrategy createObjectUnderTest( pluginMetrics, Integer.MAX_VALUE, bulkRequestSupplier, - pluginSetting); + PIPELINE_NAME, + PLUGIN_NAME); } public BulkRetryStrategy createObjectUnderTest( @@ -139,7 +140,8 @@ public BulkRetryStrategy createObjectUnderTest( pluginMetrics, maxRetries, bulkRequestSupplier, - pluginSetting); + PIPELINE_NAME, + PLUGIN_NAME); } @Test From d1cb3fa9d923c3388f53bef1c75edf7bc67d0c9f Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Thu, 12 Dec 2024 16:24:03 -0800 Subject: [PATCH 05/62] Rework OpenSearchSinkConfigurationTests.java Signed-off-by: Maxwell Brown --- .../sink/opensearch/OpenSearchSinkConfig.java | 5 +- .../OpenSearchSinkConfiguration.java | 4 + .../configuration/ActionConfiguration.java | 4 +- .../OpenSearchSinkConfigurationTests.java | 135 ++++++------------ .../create-action-config.yaml | 10 ++ .../create-action-with-expression-config.yaml | 10 ++ .../invalid-action-config.yaml | 10 ++ ...invalid-action-with-expression-config.yaml | 10 ++ .../invalid-actions-config.yaml | 11 ++ ...nvalid-actions-with-expression-config.yaml | 11 ++ .../valid_sink_config.yaml | 9 ++ 11 files changed, 122 insertions(+), 97 deletions(-) create mode 100644 data-prepper-plugins/opensearch/src/test/resources/test-configurations/create-action-config.yaml create mode 100644 data-prepper-plugins/opensearch/src/test/resources/test-configurations/create-action-with-expression-config.yaml create mode 100644 data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-action-config.yaml create mode 100644 data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-action-with-expression-config.yaml create mode 100644 data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-actions-config.yaml create mode 100644 data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-actions-with-expression-config.yaml create mode 100644 data-prepper-plugins/opensearch/src/test/resources/test-configurations/valid_sink_config.yaml diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfig.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfig.java index ba02849f65..2d8791d780 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfig.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfig.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.validation.Valid; import lombok.Getter; +import org.apache.commons.lang3.EnumUtils; import org.opensearch.dataprepper.model.opensearch.OpenSearchBulkActions; import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.ActionConfiguration; import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.DlqConfiguration; @@ -182,9 +183,11 @@ void isDlqValid() { } void isActionValid() { - if (action.equals("index") || action.equals("create") || action.equals("update") || action.equals("upsert") || action.equals("delete")) { + if (EnumUtils.isValidEnumIgnoreCase(OpenSearchBulkActions.class, action)) { return; } + + System.out.println(action); throw new IllegalArgumentException("action must be one of [index, create, update, upsert, delete]"); } diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfiguration.java index 3c563ea5c0..80d3d60528 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfiguration.java @@ -39,6 +39,10 @@ private OpenSearchSinkConfiguration( this.retryConfiguration = retryConfiguration; } + public static OpenSearchSinkConfiguration readOSConfig(final OpenSearchSinkConfig openSearchSinkConfig) { + return readOSConfig(openSearchSinkConfig, null); + } + public static OpenSearchSinkConfiguration readOSConfig(final OpenSearchSinkConfig openSearchSinkConfig, final ExpressionEvaluator expressionEvaluator) { openSearchSinkConfig.validateConfig(); final ConnectionConfiguration connectionConfiguration = diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ActionConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ActionConfiguration.java index 8c53e0d87c..4f1315d6e6 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ActionConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ActionConfiguration.java @@ -3,6 +3,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.validation.constraints.AssertTrue; import lombok.Getter; +import org.apache.commons.lang3.EnumUtils; +import org.opensearch.dataprepper.model.opensearch.OpenSearchBulkActions; public class ActionConfiguration { @Getter @@ -15,7 +17,7 @@ public class ActionConfiguration { @AssertTrue(message = "type must be one of index, create, update, upsert, delete") boolean isTypeValid() { - if (type.equals("index") || type.equals("create") || type.equals("update") || type.equals("upsert") || type.equals("delete")) { + if (EnumUtils.isValidEnumIgnoreCase(OpenSearchBulkActions.class, type)) { return true; } return false; diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfigurationTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfigurationTests.java index cb9dfbe898..b83fc67761 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfigurationTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfigurationTests.java @@ -5,17 +5,14 @@ package org.opensearch.dataprepper.plugins.sink.opensearch; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import org.junit.Test; import org.opensearch.dataprepper.expression.ExpressionEvaluator; -import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.opensearch.OpenSearchBulkActions; -import org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration; -import org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexType; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; +import java.io.File; +import java.io.IOException; import java.util.Map; import static org.junit.Assert.assertEquals; @@ -25,15 +22,23 @@ import static org.mockito.Mockito.when; public class OpenSearchSinkConfigurationTests { - private final List TEST_HOSTS = Collections.singletonList("http://localhost:9200"); + private static final String VALID_SINK_CONFIG = "test-configurations/valid_sink_config.yaml"; + private static final String INVALID_ACTION_CONFIG = "test-configurations/invalid-action-config.yaml"; + private static final String INVALID_ACTIONS_CONFIG = "test-configurations/invalid-actions-config.yaml"; + private static final String INVALID_ACTION_WITH_EXPRESSION_CONFIG = "test-configurations/invalid-action-with-expression-config.yaml"; + private static final String INVALID_ACTIONS_WITH_EXPRESSION_CONFIG = "test-configurations/invalid-actions-with-expression-config.yaml"; + private static final String CREATE_ACTION_CONFIG = "test-configurations/create-action-config.yaml"; + private static final String CREATE_ACTION_WITH_EXPRESSION_CONFIG = "test-configurations/create-action-with-expression-config.yaml"; private static final String PLUGIN_NAME = "opensearch"; private static final String PIPELINE_NAME = "integTestPipeline"; private ExpressionEvaluator expressionEvaluator; + ObjectMapper objectMapper; + @Test - public void testReadESConfig() { - final OpenSearchSinkConfiguration openSearchSinkConfiguration = OpenSearchSinkConfiguration.readESConfig( - generatePluginSetting()); + public void testReadESConfig() throws IOException { + final OpenSearchSinkConfiguration openSearchSinkConfiguration = OpenSearchSinkConfiguration.readOSConfig( + generateOpenSearchSourceConfig(VALID_SINK_CONFIG)); assertNotNull(openSearchSinkConfiguration.getConnectionConfiguration()); assertNotNull(openSearchSinkConfiguration.getIndexConfiguration()); assertNotNull(openSearchSinkConfiguration.getRetryConfiguration()); @@ -41,88 +46,34 @@ public void testReadESConfig() { } @Test(expected = IllegalArgumentException.class) - public void testInvalidAction() { - - final Map metadata = new HashMap<>(); - metadata.put(IndexConfiguration.INDEX_TYPE, IndexType.TRACE_ANALYTICS_RAW.getValue()); - metadata.put(IndexConfiguration.ACTION, "invalid"); - metadata.put(ConnectionConfiguration.HOSTS, TEST_HOSTS); - - final PluginSetting pluginSetting = new PluginSetting(PLUGIN_NAME, metadata); - pluginSetting.setPipelineName(PIPELINE_NAME); - - OpenSearchSinkConfiguration.readESConfig(pluginSetting); - + public void testInvalidAction() throws IOException { + OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSourceConfig(INVALID_ACTION_CONFIG)); } @Test(expected = IllegalArgumentException.class) - public void testInvalidActions() { - - final Map metadata = new HashMap<>(); - metadata.put(IndexConfiguration.INDEX_TYPE, IndexType.TRACE_ANALYTICS_RAW.getValue()); - List> invalidActionList = new ArrayList<>(); - Map actionMap = new HashMap<>(); - actionMap.put("type", "invalid"); - invalidActionList.add(actionMap); - metadata.put(IndexConfiguration.ACTIONS, invalidActionList); - metadata.put(ConnectionConfiguration.HOSTS, TEST_HOSTS); - - final PluginSetting pluginSetting = new PluginSetting(PLUGIN_NAME, metadata); - pluginSetting.setPipelineName(PIPELINE_NAME); - - OpenSearchSinkConfiguration.readESConfig(pluginSetting); + public void testInvalidActions() throws IOException { + OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSourceConfig(INVALID_ACTIONS_CONFIG)); } @Test(expected = IllegalArgumentException.class) - public void testInvalidActionWithExpression() { - - final Map metadata = new HashMap<>(); - metadata.put(IndexConfiguration.INDEX_TYPE, IndexType.TRACE_ANALYTICS_RAW.getValue()); - metadata.put(IndexConfiguration.ACTION, "${anInvalidFunction()}"); - metadata.put(ConnectionConfiguration.HOSTS, TEST_HOSTS); - - final PluginSetting pluginSetting = new PluginSetting(PLUGIN_NAME, metadata); - pluginSetting.setPipelineName(PIPELINE_NAME); - + public void testInvalidActionWithExpression() throws IOException { expressionEvaluator = mock(ExpressionEvaluator.class); when(expressionEvaluator.isValidExpressionStatement(anyString())).thenReturn(false); - OpenSearchSinkConfiguration.readESConfig(pluginSetting, expressionEvaluator); + OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSourceConfig(INVALID_ACTION_WITH_EXPRESSION_CONFIG), expressionEvaluator); } @Test(expected = IllegalArgumentException.class) - public void testInvalidActionsWithExpression() { - - final Map metadata = new HashMap<>(); - metadata.put(IndexConfiguration.INDEX_TYPE, IndexType.TRACE_ANALYTICS_RAW.getValue()); - List> invalidActionList = new ArrayList<>(); - Map actionMap = new HashMap<>(); - actionMap.put("type", "${anInvalidFunction()}"); - invalidActionList.add(actionMap); - metadata.put(IndexConfiguration.ACTIONS, invalidActionList); - metadata.put(ConnectionConfiguration.HOSTS, TEST_HOSTS); - - final PluginSetting pluginSetting = new PluginSetting(PLUGIN_NAME, metadata); - pluginSetting.setPipelineName(PIPELINE_NAME); - + public void testInvalidActionsWithExpression() throws IOException { expressionEvaluator = mock(ExpressionEvaluator.class); when(expressionEvaluator.isValidExpressionStatement(anyString())).thenReturn(false); - OpenSearchSinkConfiguration.readESConfig(pluginSetting, expressionEvaluator); + OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSourceConfig(INVALID_ACTIONS_WITH_EXPRESSION_CONFIG), expressionEvaluator); } @Test - public void testReadESConfigWithBulkActionCreate() { - - final Map metadata = new HashMap<>(); - metadata.put(IndexConfiguration.INDEX_TYPE, IndexType.TRACE_ANALYTICS_RAW.getValue()); - metadata.put(IndexConfiguration.ACTION, OpenSearchBulkActions.CREATE.toString()); - metadata.put(ConnectionConfiguration.HOSTS, TEST_HOSTS); - - final PluginSetting pluginSetting = new PluginSetting(PLUGIN_NAME, metadata); - pluginSetting.setPipelineName(PIPELINE_NAME); - + public void testReadOSConfigWithBulkActionCreate() throws IOException { final OpenSearchSinkConfiguration openSearchSinkConfiguration = - OpenSearchSinkConfiguration.readESConfig(pluginSetting); + OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSourceConfig(CREATE_ACTION_CONFIG)); assertNotNull(openSearchSinkConfiguration.getConnectionConfiguration()); assertNotNull(openSearchSinkConfiguration.getIndexConfiguration()); @@ -130,34 +81,28 @@ public void testReadESConfigWithBulkActionCreate() { } @Test - public void testReadESConfigWithBulkActionCreateExpression() { - - final String actionFormatExpression = "${getMetadata(\"action\")}"; - final Map metadata = new HashMap<>(); - metadata.put(IndexConfiguration.INDEX_TYPE, IndexType.TRACE_ANALYTICS_RAW.getValue()); - metadata.put(IndexConfiguration.ACTION, actionFormatExpression); - metadata.put(ConnectionConfiguration.HOSTS, TEST_HOSTS); - - final PluginSetting pluginSetting = new PluginSetting(PLUGIN_NAME, metadata); - pluginSetting.setPipelineName(PIPELINE_NAME); + public void testReadESConfigWithBulkActionCreateExpression() throws IOException { expressionEvaluator = mock(ExpressionEvaluator.class); - when(expressionEvaluator.isValidFormatExpression(actionFormatExpression)).thenReturn(true); + when(expressionEvaluator.isValidFormatExpression("${getMetadata(\"action\")}")).thenReturn(true); + final OpenSearchSinkConfiguration openSearchSinkConfiguration = - OpenSearchSinkConfiguration.readESConfig(pluginSetting, expressionEvaluator); + OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSourceConfig(CREATE_ACTION_CONFIG)); assertNotNull(openSearchSinkConfiguration.getConnectionConfiguration()); assertNotNull(openSearchSinkConfiguration.getIndexConfiguration()); assertNotNull(openSearchSinkConfiguration.getRetryConfiguration()); } - private PluginSetting generatePluginSetting() { - final Map metadata = new HashMap<>(); - metadata.put(IndexConfiguration.INDEX_TYPE, IndexType.TRACE_ANALYTICS_RAW.getValue()); - metadata.put(ConnectionConfiguration.HOSTS, TEST_HOSTS); + private OpenSearchSinkConfig generateOpenSearchSourceConfig(String yamlFile) throws IOException { + final File configurationFile = new File(getClass().getClassLoader().getResource(yamlFile).getFile()); + objectMapper = new ObjectMapper(new YAMLFactory()); + final Map pipelineConfig = objectMapper.readValue(configurationFile, Map.class); + final Map sinkMap = (Map) pipelineConfig.get("sink"); + final Map opensearchSinkMap = (Map) sinkMap.get("opensearch"); + String json = objectMapper.writeValueAsString(opensearchSinkMap); + OpenSearchSinkConfig openSearchSinkConfig = objectMapper.readValue(json, OpenSearchSinkConfig.class); - final PluginSetting pluginSetting = new PluginSetting(PLUGIN_NAME, metadata); - pluginSetting.setPipelineName(PIPELINE_NAME); - return pluginSetting; + return openSearchSinkConfig; } -} +} \ No newline at end of file diff --git a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/create-action-config.yaml b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/create-action-config.yaml new file mode 100644 index 0000000000..ec048e6d93 --- /dev/null +++ b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/create-action-config.yaml @@ -0,0 +1,10 @@ +sink: + opensearch: + hosts: [ "http://localhost:9200" ] + index: "no-more-plugin-setting" + index_type: "trace-analytics-raw" + action: "update" + aws: + sts_role_arn: "arn:aws:iam::867344451130:role/OSIJiraPipeline" + region: "us-east-2" + serverless: true \ No newline at end of file diff --git a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/create-action-with-expression-config.yaml b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/create-action-with-expression-config.yaml new file mode 100644 index 0000000000..e04f328d3f --- /dev/null +++ b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/create-action-with-expression-config.yaml @@ -0,0 +1,10 @@ +sink: + opensearch: + hosts: [ "http://localhost:9200" ] + index: "no-more-plugin-setting" + index_type: "trace-analytics-raw" + action: "${getMetadata(\"action\")}" + aws: + sts_role_arn: "arn:aws:iam::867344451130:role/OSIJiraPipeline" + region: "us-east-2" + serverless: true \ No newline at end of file diff --git a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-action-config.yaml b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-action-config.yaml new file mode 100644 index 0000000000..800cc5c753 --- /dev/null +++ b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-action-config.yaml @@ -0,0 +1,10 @@ +sink: + opensearch: + hosts: [ "http://localhost:9200" ] + index: "no-more-plugin-setting" + index_type: "trace-analytics-raw" + action: "invalid" + aws: + sts_role_arn: "arn:aws:iam::867344451130:role/OSIJiraPipeline" + region: "us-east-2" + serverless: true \ No newline at end of file diff --git a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-action-with-expression-config.yaml b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-action-with-expression-config.yaml new file mode 100644 index 0000000000..6127f473c8 --- /dev/null +++ b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-action-with-expression-config.yaml @@ -0,0 +1,10 @@ +sink: + opensearch: + hosts: [ "http://localhost:9200" ] + index: "no-more-plugin-setting" + index_type: "trace-analytics-raw" + action: "${anInvalidFunction()}" + aws: + sts_role_arn: "arn:aws:iam::867344451130:role/OSIJiraPipeline" + region: "us-east-2" + serverless: true \ No newline at end of file diff --git a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-actions-config.yaml b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-actions-config.yaml new file mode 100644 index 0000000000..80ce9cf0b6 --- /dev/null +++ b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-actions-config.yaml @@ -0,0 +1,11 @@ +sink: + opensearch: + hosts: [ "http://localhost:9200" ] + index: "no-more-plugin-setting" + index_type: "trace-analytics-raw" + actions: + - type: "invalid" + aws: + sts_role_arn: "arn:aws:iam::867344451130:role/OSIJiraPipeline" + region: "us-east-2" + serverless: true \ No newline at end of file diff --git a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-actions-with-expression-config.yaml b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-actions-with-expression-config.yaml new file mode 100644 index 0000000000..f94836cece --- /dev/null +++ b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-actions-with-expression-config.yaml @@ -0,0 +1,11 @@ +sink: + opensearch: + hosts: [ "http://localhost:9200" ] + index: "no-more-plugin-setting" + index_type: "trace-analytics-raw" + actions: + - type: "${anInvalidFunction()}" + aws: + sts_role_arn: "arn:aws:iam::867344451130:role/OSIJiraPipeline" + region: "us-east-2" + serverless: true \ No newline at end of file diff --git a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/valid_sink_config.yaml b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/valid_sink_config.yaml new file mode 100644 index 0000000000..ea8ebb564a --- /dev/null +++ b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/valid_sink_config.yaml @@ -0,0 +1,9 @@ +sink: + opensearch: + hosts: [ "http://localhost:9200" ] + index: "no-more-plugin-setting" + index_type: "trace-analytics-raw" + aws: + sts_role_arn: "arn:aws:iam::867344451130:role/OSIJiraPipeline" + region: "us-east-2" + serverless: true \ No newline at end of file From 40f62667d48bc28091ccdcc374dd3dcb785a1878 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Thu, 12 Dec 2024 17:25:07 -0800 Subject: [PATCH 06/62] Rework RetryConfigurationTests.java Signed-off-by: Maxwell Brown --- .../sink/opensearch/OpenSearchSinkConfig.java | 1 - .../OpenSearchSinkConfigurationTests.java | 4 +- .../opensearch/RetryConfigurationTests.java | 71 ++++++++++--------- ...reate-actions-with-expression-config.yaml} | 4 +- .../dlq-file-and-dlq-plugin-config.yaml | 9 +++ .../dlq-file-path-10-retries-config.yaml | 6 ++ .../invalid-max-retries-config.yaml | 6 ++ .../no-dlq-file-path-config.yaml | 6 ++ .../with-dlq-plugin-10-retries-config.yaml | 9 +++ 9 files changed, 78 insertions(+), 38 deletions(-) rename data-prepper-plugins/opensearch/src/test/resources/test-configurations/{create-action-with-expression-config.yaml => create-actions-with-expression-config.yaml} (76%) create mode 100644 data-prepper-plugins/opensearch/src/test/resources/test-configurations/dlq-file-and-dlq-plugin-config.yaml create mode 100644 data-prepper-plugins/opensearch/src/test/resources/test-configurations/dlq-file-path-10-retries-config.yaml create mode 100644 data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-max-retries-config.yaml create mode 100644 data-prepper-plugins/opensearch/src/test/resources/test-configurations/no-dlq-file-path-config.yaml create mode 100644 data-prepper-plugins/opensearch/src/test/resources/test-configurations/with-dlq-plugin-10-retries-config.yaml diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfig.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfig.java index 2d8791d780..b5a32f2d5c 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfig.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfig.java @@ -168,7 +168,6 @@ public Long getFlushTimeout(Long defaultFlushTimeout) { private DlqConfiguration dlq; - public void validateConfig() { isActionValid(); isDlqValid(); diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfigurationTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfigurationTests.java index b83fc67761..6d8822d337 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfigurationTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfigurationTests.java @@ -28,7 +28,7 @@ public class OpenSearchSinkConfigurationTests { private static final String INVALID_ACTION_WITH_EXPRESSION_CONFIG = "test-configurations/invalid-action-with-expression-config.yaml"; private static final String INVALID_ACTIONS_WITH_EXPRESSION_CONFIG = "test-configurations/invalid-actions-with-expression-config.yaml"; private static final String CREATE_ACTION_CONFIG = "test-configurations/create-action-config.yaml"; - private static final String CREATE_ACTION_WITH_EXPRESSION_CONFIG = "test-configurations/create-action-with-expression-config.yaml"; + private static final String CREATE_ACTIONS_WITH_EXPRESSION_CONFIG = "test-configurations/create-actions-with-expression-config.yaml"; private static final String PLUGIN_NAME = "opensearch"; private static final String PIPELINE_NAME = "integTestPipeline"; private ExpressionEvaluator expressionEvaluator; @@ -87,7 +87,7 @@ public void testReadESConfigWithBulkActionCreateExpression() throws IOException when(expressionEvaluator.isValidFormatExpression("${getMetadata(\"action\")}")).thenReturn(true); final OpenSearchSinkConfiguration openSearchSinkConfiguration = - OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSourceConfig(CREATE_ACTION_CONFIG)); + OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSourceConfig(CREATE_ACTIONS_WITH_EXPRESSION_CONFIG)); assertNotNull(openSearchSinkConfiguration.getConnectionConfiguration()); assertNotNull(openSearchSinkConfiguration.getIndexConfiguration()); diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfigurationTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfigurationTests.java index e5b066234f..9498f11315 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfigurationTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfigurationTests.java @@ -5,19 +5,32 @@ package org.opensearch.dataprepper.plugins.sink.opensearch; -import org.opensearch.dataprepper.model.configuration.PluginSetting; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import org.junit.Test; +import org.opensearch.dataprepper.plugins.dlq.s3.S3DlqWriterConfig; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.DlqConfiguration; -import java.util.HashMap; -import java.util.LinkedHashMap; +import java.io.File; +import java.io.IOException; import java.util.Map; +import java.util.Optional; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; public class RetryConfigurationTests { + private static final String INVALID_MAX_RETRIES_CONFIG = "test-configurations/invalid-max-retries-config.yaml"; + private static final String NO_DLQ_FILE_PATH = "test-configurations/no-dlq-file-path-config.yaml"; + private static final String DLQ_FILE_PATH_10_RETRIES = "test-configurations/dlq-file-path-10-retries-config.yaml"; + private static final String WITH_DLQ_PLUGIN_10_RETRIES = "test-configurations/with-dlq-plugin-10-retries-config.yaml"; + private static final String DLQ_FILE_AND_DLQ_PLUGIN = "test-configurations/dlq-file-and-dlq-plugin-config.yaml"; + + ObjectMapper objectMapper; + @Test public void testDefaultConfigurationIsNotNull() { final RetryConfiguration retryConfiguration = new RetryConfiguration.Builder().build(); @@ -26,62 +39,52 @@ public void testDefaultConfigurationIsNotNull() { } @Test - public void testReadRetryConfigInvalidMaxRetries() { - final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(generatePluginSetting(null, -1, null)); + public void testReadRetryConfigInvalidMaxRetries() throws IOException { + final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(generateOpenSearchSourceConfig(INVALID_MAX_RETRIES_CONFIG)); assertThrows(IllegalArgumentException.class, () -> retryConfiguration.getMaxRetries()); } @Test - public void testReadRetryConfigNoDLQFilePath() { - final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(generatePluginSetting(null, null, null)); + public void testReadRetryConfigNoDLQFilePath() throws IOException { + final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(generateOpenSearchSourceConfig(NO_DLQ_FILE_PATH)); assertNull(retryConfiguration.getDlqFile()); assertEquals(retryConfiguration.getMaxRetries(), Integer.MAX_VALUE); assertFalse(retryConfiguration.getDlq().isPresent()); } @Test - public void testReadRetryConfigWithDLQFilePath() { + public void testReadRetryConfigWithDLQFilePath() throws IOException { final String fakeDlqFilePath = "foo.txt"; final int maxRetries = 10; - final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(generatePluginSetting(fakeDlqFilePath, maxRetries, null)); + final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(generateOpenSearchSourceConfig(DLQ_FILE_PATH_10_RETRIES)); assertEquals(fakeDlqFilePath, retryConfiguration.getDlqFile()); assertEquals(maxRetries, retryConfiguration.getMaxRetries()); } @Test - public void testReadRetryConfigWithDLQPlugin() { - final Map fakePlugin = new LinkedHashMap<>(); - final Map lowLevelPluginSettings = new HashMap<>(); - lowLevelPluginSettings.put("field1", "value1"); - lowLevelPluginSettings.put("field2", "value2"); - fakePlugin.put("another_dlq", lowLevelPluginSettings); + public void testReadRetryConfigWithDLQPlugin() throws IOException { final int maxRetries = 10; - final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(generatePluginSetting(null, maxRetries, fakePlugin)); - assertEquals("another_dlq", retryConfiguration.getDlq().get().getPluginName()); + final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(generateOpenSearchSourceConfig(WITH_DLQ_PLUGIN_10_RETRIES)); + Optional dlqConfiguration = retryConfiguration.getDlq(); + S3DlqWriterConfig s3DlqWriterConfig = dlqConfiguration.get().getS3DlqWriterConfig(); + assertInstanceOf(S3DlqWriterConfig.class, dlqConfiguration.get().getS3DlqWriterConfig()); assertEquals(maxRetries, retryConfiguration.getMaxRetries()); } @Test public void testReadRetryConfigWithDLQPluginAndDLQFilePath() { - final String fakeDlqFilePath = "foo.txt"; - final Map fakePlugin = new LinkedHashMap<>(); - fakePlugin.put("another_dlq", "value1"); - final int maxRetries = 10; - assertThrows(RuntimeException.class, () -> RetryConfiguration.readRetryConfig(generatePluginSetting(fakeDlqFilePath, maxRetries, fakePlugin))); + assertThrows(RuntimeException.class, () -> OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSourceConfig(DLQ_FILE_AND_DLQ_PLUGIN))); } - private PluginSetting generatePluginSetting(final String dlqFilePath, final Integer maxRetries, final Map pluginSettings) { - final Map metadata = new HashMap<>(); - if (dlqFilePath != null) { - metadata.put(RetryConfiguration.DLQ_FILE, dlqFilePath); - } - if (maxRetries != null) { - metadata.put(RetryConfiguration.MAX_RETRIES, maxRetries); - } - if (pluginSettings != null) { - metadata.put(RetryConfiguration.DLQ, pluginSettings); - } + private OpenSearchSinkConfig generateOpenSearchSourceConfig(String yamlFile) throws IOException { + final File configurationFile = new File(getClass().getClassLoader().getResource(yamlFile).getFile()); + objectMapper = new ObjectMapper(new YAMLFactory()); + final Map pipelineConfig = objectMapper.readValue(configurationFile, Map.class); + final Map sinkMap = (Map) pipelineConfig.get("sink"); + final Map opensearchSinkMap = (Map) sinkMap.get("opensearch"); + String json = objectMapper.writeValueAsString(opensearchSinkMap); + OpenSearchSinkConfig openSearchSinkConfig = objectMapper.readValue(json, OpenSearchSinkConfig.class); - return new PluginSetting("opensearch", metadata); + return openSearchSinkConfig; } } diff --git a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/create-action-with-expression-config.yaml b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/create-actions-with-expression-config.yaml similarity index 76% rename from data-prepper-plugins/opensearch/src/test/resources/test-configurations/create-action-with-expression-config.yaml rename to data-prepper-plugins/opensearch/src/test/resources/test-configurations/create-actions-with-expression-config.yaml index e04f328d3f..7cd109e734 100644 --- a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/create-action-with-expression-config.yaml +++ b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/create-actions-with-expression-config.yaml @@ -3,7 +3,9 @@ sink: hosts: [ "http://localhost:9200" ] index: "no-more-plugin-setting" index_type: "trace-analytics-raw" - action: "${getMetadata(\"action\")}" + actions: + - type: "create" + when: "${getMetadata(\"action\")}" aws: sts_role_arn: "arn:aws:iam::867344451130:role/OSIJiraPipeline" region: "us-east-2" diff --git a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/dlq-file-and-dlq-plugin-config.yaml b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/dlq-file-and-dlq-plugin-config.yaml new file mode 100644 index 0000000000..94444597e1 --- /dev/null +++ b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/dlq-file-and-dlq-plugin-config.yaml @@ -0,0 +1,9 @@ +sink: + opensearch: + hosts: [ "http://localhost:9200" ] + dlq: + s3: + bucket: "my-dlq-bucket" + key_path_prefix: "dlq-files/" + dlq_file: foo.txt + max_retries: 10 \ No newline at end of file diff --git a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/dlq-file-path-10-retries-config.yaml b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/dlq-file-path-10-retries-config.yaml new file mode 100644 index 0000000000..e1e1bfa2bd --- /dev/null +++ b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/dlq-file-path-10-retries-config.yaml @@ -0,0 +1,6 @@ +sink: + opensearch: + hosts: [ "http://localhost:9200" ] + dlq: null + dlq_file: foo.txt + max_retries: 10 \ No newline at end of file diff --git a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-max-retries-config.yaml b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-max-retries-config.yaml new file mode 100644 index 0000000000..8452700433 --- /dev/null +++ b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-max-retries-config.yaml @@ -0,0 +1,6 @@ +sink: + opensearch: + hosts: [ "http://localhost:9200" ] + dlq: null + dlq_file: null + max_retries: -1 \ No newline at end of file diff --git a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/no-dlq-file-path-config.yaml b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/no-dlq-file-path-config.yaml new file mode 100644 index 0000000000..6e7bec4427 --- /dev/null +++ b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/no-dlq-file-path-config.yaml @@ -0,0 +1,6 @@ +sink: + opensearch: + hosts: [ "http://localhost:9200" ] + dlq: null + dlq_file: null + max_retries: null \ No newline at end of file diff --git a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/with-dlq-plugin-10-retries-config.yaml b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/with-dlq-plugin-10-retries-config.yaml new file mode 100644 index 0000000000..f8bd8f15b5 --- /dev/null +++ b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/with-dlq-plugin-10-retries-config.yaml @@ -0,0 +1,9 @@ +sink: + opensearch: + hosts: [ "http://localhost:9200" ] + dlq: + s3: + bucket: "my-dlq-bucket" + key_path_prefix: "dlq-files/" + dlq_file: null + max_retries: 10 \ No newline at end of file From b47c0da4004af8dabf6c1b14c841fa1823b42ccd Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Thu, 12 Dec 2024 17:52:22 -0800 Subject: [PATCH 07/62] small changes Signed-off-by: Maxwell Brown --- .../plugins/sink/opensearch/RetryConfiguration.java | 5 ----- .../plugins/sink/opensearch/RetryConfigurationTests.java | 1 - 2 files changed, 6 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfiguration.java index ef9ec9ade8..f7269e3683 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfiguration.java @@ -86,9 +86,4 @@ public static RetryConfiguration readRetryConfig(final OpenSearchSinkConfig open } return builder.build(); } - - public static RetryConfiguration readRetryConfig(final PluginSetting pluginSetting) { - RetryConfiguration.Builder builder = new RetryConfiguration.Builder(); - return builder.build(); - } } diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfigurationTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfigurationTests.java index 9498f11315..6cfcf1716e 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfigurationTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfigurationTests.java @@ -66,7 +66,6 @@ public void testReadRetryConfigWithDLQPlugin() throws IOException { final int maxRetries = 10; final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(generateOpenSearchSourceConfig(WITH_DLQ_PLUGIN_10_RETRIES)); Optional dlqConfiguration = retryConfiguration.getDlq(); - S3DlqWriterConfig s3DlqWriterConfig = dlqConfiguration.get().getS3DlqWriterConfig(); assertInstanceOf(S3DlqWriterConfig.class, dlqConfiguration.get().getS3DlqWriterConfig()); assertEquals(maxRetries, retryConfiguration.getMaxRetries()); } From ce076725f7696cf092a98a0703a5ac898511f4d5 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Thu, 12 Dec 2024 17:53:43 -0800 Subject: [PATCH 08/62] small changes Signed-off-by: Maxwell Brown --- .../sink/opensearch/OpenSearchSinkConfigurationTests.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfigurationTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfigurationTests.java index 6d8822d337..7c69e72e24 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfigurationTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfigurationTests.java @@ -29,8 +29,6 @@ public class OpenSearchSinkConfigurationTests { private static final String INVALID_ACTIONS_WITH_EXPRESSION_CONFIG = "test-configurations/invalid-actions-with-expression-config.yaml"; private static final String CREATE_ACTION_CONFIG = "test-configurations/create-action-config.yaml"; private static final String CREATE_ACTIONS_WITH_EXPRESSION_CONFIG = "test-configurations/create-actions-with-expression-config.yaml"; - private static final String PLUGIN_NAME = "opensearch"; - private static final String PIPELINE_NAME = "integTestPipeline"; private ExpressionEvaluator expressionEvaluator; ObjectMapper objectMapper; From 19dc50ef7d545440bff25dda18fef121ccfbdc00 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Fri, 13 Dec 2024 15:07:58 -0800 Subject: [PATCH 09/62] condense many test yaml files into one Signed-off-by: Maxwell Brown --- .../OpenSearchSinkConfigurationTests.java | 38 +++--- .../opensearch/RetryConfigurationTests.java | 28 ++-- .../open-search-sink-configurations.yaml | 121 ++++++++++++++++++ .../create-action-config.yaml | 10 -- ...create-actions-with-expression-config.yaml | 12 -- .../dlq-file-path-10-retries-config.yaml | 6 - .../invalid-action-config.yaml | 10 -- ...invalid-action-with-expression-config.yaml | 10 -- .../invalid-actions-config.yaml | 11 -- ...nvalid-actions-with-expression-config.yaml | 11 -- .../invalid-max-retries-config.yaml | 6 - .../no-dlq-file-path-config.yaml | 6 - .../valid_sink_config.yaml | 9 -- .../with-dlq-plugin-10-retries-config.yaml | 9 -- 14 files changed, 157 insertions(+), 130 deletions(-) create mode 100644 data-prepper-plugins/opensearch/src/test/resources/open-search-sink-configurations.yaml delete mode 100644 data-prepper-plugins/opensearch/src/test/resources/test-configurations/create-action-config.yaml delete mode 100644 data-prepper-plugins/opensearch/src/test/resources/test-configurations/create-actions-with-expression-config.yaml delete mode 100644 data-prepper-plugins/opensearch/src/test/resources/test-configurations/dlq-file-path-10-retries-config.yaml delete mode 100644 data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-action-config.yaml delete mode 100644 data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-action-with-expression-config.yaml delete mode 100644 data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-actions-config.yaml delete mode 100644 data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-actions-with-expression-config.yaml delete mode 100644 data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-max-retries-config.yaml delete mode 100644 data-prepper-plugins/opensearch/src/test/resources/test-configurations/no-dlq-file-path-config.yaml delete mode 100644 data-prepper-plugins/opensearch/src/test/resources/test-configurations/valid_sink_config.yaml delete mode 100644 data-prepper-plugins/opensearch/src/test/resources/test-configurations/with-dlq-plugin-10-retries-config.yaml diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfigurationTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfigurationTests.java index 7c69e72e24..12acd8053a 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfigurationTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfigurationTests.java @@ -22,13 +22,14 @@ import static org.mockito.Mockito.when; public class OpenSearchSinkConfigurationTests { - private static final String VALID_SINK_CONFIG = "test-configurations/valid_sink_config.yaml"; - private static final String INVALID_ACTION_CONFIG = "test-configurations/invalid-action-config.yaml"; - private static final String INVALID_ACTIONS_CONFIG = "test-configurations/invalid-actions-config.yaml"; - private static final String INVALID_ACTION_WITH_EXPRESSION_CONFIG = "test-configurations/invalid-action-with-expression-config.yaml"; - private static final String INVALID_ACTIONS_WITH_EXPRESSION_CONFIG = "test-configurations/invalid-actions-with-expression-config.yaml"; - private static final String CREATE_ACTION_CONFIG = "test-configurations/create-action-config.yaml"; - private static final String CREATE_ACTIONS_WITH_EXPRESSION_CONFIG = "test-configurations/create-actions-with-expression-config.yaml"; + private static final String OPEN_SEARCH_SINK_CONFIGURATIONS = "open-search-sink-configurations.yaml"; + private static final String VALID_SINK_CONFIG = "valid-sink"; + private static final String INVALID_ACTION_CONFIG = "invalid-action"; + private static final String INVALID_ACTIONS_CONFIG = "invalid-actions"; + private static final String INVALID_ACTION_WITH_EXPRESSION_CONFIG = "invalid-action-with-expression"; + private static final String INVALID_ACTIONS_WITH_EXPRESSION_CONFIG = "invalid-actions-with-expression"; + private static final String CREATE_ACTION_CONFIG = "create-action"; + private static final String CREATE_ACTIONS_WITH_EXPRESSION_CONFIG = "create-actions-with-expression"; private ExpressionEvaluator expressionEvaluator; ObjectMapper objectMapper; @@ -36,7 +37,7 @@ public class OpenSearchSinkConfigurationTests { @Test public void testReadESConfig() throws IOException { final OpenSearchSinkConfiguration openSearchSinkConfiguration = OpenSearchSinkConfiguration.readOSConfig( - generateOpenSearchSourceConfig(VALID_SINK_CONFIG)); + generateOpenSearchSinkConfig(VALID_SINK_CONFIG)); assertNotNull(openSearchSinkConfiguration.getConnectionConfiguration()); assertNotNull(openSearchSinkConfiguration.getIndexConfiguration()); assertNotNull(openSearchSinkConfiguration.getRetryConfiguration()); @@ -45,12 +46,12 @@ public void testReadESConfig() throws IOException { @Test(expected = IllegalArgumentException.class) public void testInvalidAction() throws IOException { - OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSourceConfig(INVALID_ACTION_CONFIG)); + OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSinkConfig(INVALID_ACTION_CONFIG)); } @Test(expected = IllegalArgumentException.class) public void testInvalidActions() throws IOException { - OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSourceConfig(INVALID_ACTIONS_CONFIG)); + OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSinkConfig(INVALID_ACTIONS_CONFIG)); } @@ -58,20 +59,20 @@ public void testInvalidActions() throws IOException { public void testInvalidActionWithExpression() throws IOException { expressionEvaluator = mock(ExpressionEvaluator.class); when(expressionEvaluator.isValidExpressionStatement(anyString())).thenReturn(false); - OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSourceConfig(INVALID_ACTION_WITH_EXPRESSION_CONFIG), expressionEvaluator); + OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSinkConfig(INVALID_ACTION_WITH_EXPRESSION_CONFIG), expressionEvaluator); } @Test(expected = IllegalArgumentException.class) public void testInvalidActionsWithExpression() throws IOException { expressionEvaluator = mock(ExpressionEvaluator.class); when(expressionEvaluator.isValidExpressionStatement(anyString())).thenReturn(false); - OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSourceConfig(INVALID_ACTIONS_WITH_EXPRESSION_CONFIG), expressionEvaluator); + OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSinkConfig(INVALID_ACTIONS_WITH_EXPRESSION_CONFIG), expressionEvaluator); } @Test public void testReadOSConfigWithBulkActionCreate() throws IOException { final OpenSearchSinkConfiguration openSearchSinkConfiguration = - OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSourceConfig(CREATE_ACTION_CONFIG)); + OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSinkConfig(CREATE_ACTION_CONFIG)); assertNotNull(openSearchSinkConfiguration.getConnectionConfiguration()); assertNotNull(openSearchSinkConfiguration.getIndexConfiguration()); @@ -85,17 +86,19 @@ public void testReadESConfigWithBulkActionCreateExpression() throws IOException when(expressionEvaluator.isValidFormatExpression("${getMetadata(\"action\")}")).thenReturn(true); final OpenSearchSinkConfiguration openSearchSinkConfiguration = - OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSourceConfig(CREATE_ACTIONS_WITH_EXPRESSION_CONFIG)); + OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSinkConfig(CREATE_ACTIONS_WITH_EXPRESSION_CONFIG)); assertNotNull(openSearchSinkConfiguration.getConnectionConfiguration()); assertNotNull(openSearchSinkConfiguration.getIndexConfiguration()); assertNotNull(openSearchSinkConfiguration.getRetryConfiguration()); } - private OpenSearchSinkConfig generateOpenSearchSourceConfig(String yamlFile) throws IOException { - final File configurationFile = new File(getClass().getClassLoader().getResource(yamlFile).getFile()); + + private OpenSearchSinkConfig generateOpenSearchSinkConfig(String pipelineName) throws IOException { + final File configurationFile = new File(getClass().getClassLoader().getResource(OPEN_SEARCH_SINK_CONFIGURATIONS).getFile()); objectMapper = new ObjectMapper(new YAMLFactory()); - final Map pipelineConfig = objectMapper.readValue(configurationFile, Map.class); + final Map pipelineConfigs = objectMapper.readValue(configurationFile, Map.class); + final Map pipelineConfig = (Map) pipelineConfigs.get(pipelineName); final Map sinkMap = (Map) pipelineConfig.get("sink"); final Map opensearchSinkMap = (Map) sinkMap.get("opensearch"); String json = objectMapper.writeValueAsString(opensearchSinkMap); @@ -103,4 +106,5 @@ private OpenSearchSinkConfig generateOpenSearchSourceConfig(String yamlFile) thr return openSearchSinkConfig; } + } \ No newline at end of file diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfigurationTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfigurationTests.java index 6cfcf1716e..3f8e4d11a8 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfigurationTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfigurationTests.java @@ -23,11 +23,12 @@ import static org.junit.jupiter.api.Assertions.assertInstanceOf; public class RetryConfigurationTests { - private static final String INVALID_MAX_RETRIES_CONFIG = "test-configurations/invalid-max-retries-config.yaml"; - private static final String NO_DLQ_FILE_PATH = "test-configurations/no-dlq-file-path-config.yaml"; - private static final String DLQ_FILE_PATH_10_RETRIES = "test-configurations/dlq-file-path-10-retries-config.yaml"; - private static final String WITH_DLQ_PLUGIN_10_RETRIES = "test-configurations/with-dlq-plugin-10-retries-config.yaml"; - private static final String DLQ_FILE_AND_DLQ_PLUGIN = "test-configurations/dlq-file-and-dlq-plugin-config.yaml"; + private static final String OPEN_SEARCH_SINK_CONFIGURATIONS = "open-search-sink-configurations.yaml"; + private static final String INVALID_MAX_RETRIES_CONFIG = "invalid-max-retries"; + private static final String NO_DLQ_FILE_PATH = "no-dlq-file-path"; + private static final String DLQ_FILE_PATH_10_RETRIES = "dlq-file-path-10-retries"; + private static final String WITH_DLQ_PLUGIN_10_RETRIES = "with-dlq-plugin-10-retries"; + private static final String DLQ_FILE_AND_DLQ_PLUGIN = "dlq-file-and-dlq-plugin"; ObjectMapper objectMapper; @@ -40,13 +41,13 @@ public void testDefaultConfigurationIsNotNull() { @Test public void testReadRetryConfigInvalidMaxRetries() throws IOException { - final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(generateOpenSearchSourceConfig(INVALID_MAX_RETRIES_CONFIG)); + final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(generateOpenSearchSinkConfig(INVALID_MAX_RETRIES_CONFIG)); assertThrows(IllegalArgumentException.class, () -> retryConfiguration.getMaxRetries()); } @Test public void testReadRetryConfigNoDLQFilePath() throws IOException { - final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(generateOpenSearchSourceConfig(NO_DLQ_FILE_PATH)); + final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(generateOpenSearchSinkConfig(NO_DLQ_FILE_PATH)); assertNull(retryConfiguration.getDlqFile()); assertEquals(retryConfiguration.getMaxRetries(), Integer.MAX_VALUE); assertFalse(retryConfiguration.getDlq().isPresent()); @@ -56,7 +57,7 @@ public void testReadRetryConfigNoDLQFilePath() throws IOException { public void testReadRetryConfigWithDLQFilePath() throws IOException { final String fakeDlqFilePath = "foo.txt"; final int maxRetries = 10; - final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(generateOpenSearchSourceConfig(DLQ_FILE_PATH_10_RETRIES)); + final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(generateOpenSearchSinkConfig(DLQ_FILE_PATH_10_RETRIES)); assertEquals(fakeDlqFilePath, retryConfiguration.getDlqFile()); assertEquals(maxRetries, retryConfiguration.getMaxRetries()); } @@ -64,7 +65,7 @@ public void testReadRetryConfigWithDLQFilePath() throws IOException { @Test public void testReadRetryConfigWithDLQPlugin() throws IOException { final int maxRetries = 10; - final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(generateOpenSearchSourceConfig(WITH_DLQ_PLUGIN_10_RETRIES)); + final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(generateOpenSearchSinkConfig(WITH_DLQ_PLUGIN_10_RETRIES)); Optional dlqConfiguration = retryConfiguration.getDlq(); assertInstanceOf(S3DlqWriterConfig.class, dlqConfiguration.get().getS3DlqWriterConfig()); assertEquals(maxRetries, retryConfiguration.getMaxRetries()); @@ -72,13 +73,14 @@ public void testReadRetryConfigWithDLQPlugin() throws IOException { @Test public void testReadRetryConfigWithDLQPluginAndDLQFilePath() { - assertThrows(RuntimeException.class, () -> OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSourceConfig(DLQ_FILE_AND_DLQ_PLUGIN))); + assertThrows(RuntimeException.class, () -> OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSinkConfig(DLQ_FILE_AND_DLQ_PLUGIN))); } - private OpenSearchSinkConfig generateOpenSearchSourceConfig(String yamlFile) throws IOException { - final File configurationFile = new File(getClass().getClassLoader().getResource(yamlFile).getFile()); + private OpenSearchSinkConfig generateOpenSearchSinkConfig(String pipelineName) throws IOException { + final File configurationFile = new File(getClass().getClassLoader().getResource(OPEN_SEARCH_SINK_CONFIGURATIONS).getFile()); objectMapper = new ObjectMapper(new YAMLFactory()); - final Map pipelineConfig = objectMapper.readValue(configurationFile, Map.class); + final Map pipelineConfigs = objectMapper.readValue(configurationFile, Map.class); + final Map pipelineConfig = (Map) pipelineConfigs.get(pipelineName); final Map sinkMap = (Map) pipelineConfig.get("sink"); final Map opensearchSinkMap = (Map) sinkMap.get("opensearch"); String json = objectMapper.writeValueAsString(opensearchSinkMap); diff --git a/data-prepper-plugins/opensearch/src/test/resources/open-search-sink-configurations.yaml b/data-prepper-plugins/opensearch/src/test/resources/open-search-sink-configurations.yaml new file mode 100644 index 0000000000..3b4b904eaf --- /dev/null +++ b/data-prepper-plugins/opensearch/src/test/resources/open-search-sink-configurations.yaml @@ -0,0 +1,121 @@ +valid-sink: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + index: "no-more-plugin-setting" + index_type: "trace-analytics-raw" + aws: + sts_role_arn: "arn:aws:iam::867344451130:role/OSIJiraPipeline" + region: "us-east-2" + serverless: true +invalid-action: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + index: "no-more-plugin-setting" + index_type: "trace-analytics-raw" + action: "invalid" + aws: + sts_role_arn: "arn:aws:iam::867344451130:role/OSIJiraPipeline" + region: "us-east-2" + serverless: true +invalid-actions: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + index: "no-more-plugin-setting" + index_type: "trace-analytics-raw" + actions: + - type: "invalid" + aws: + sts_role_arn: "arn:aws:iam::867344451130:role/OSIJiraPipeline" + region: "us-east-2" + serverless: true +invalid-action-with-expression: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + index: "no-more-plugin-setting" + index_type: "trace-analytics-raw" + action: "${anInvalidFunction()}" + aws: + sts_role_arn: "arn:aws:iam::867344451130:role/OSIJiraPipeline" + region: "us-east-2" + serverless: true +invalid-actions-with-expression: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + index: "no-more-plugin-setting" + index_type: "trace-analytics-raw" + actions: + - type: "${anInvalidFunction()}" + aws: + sts_role_arn: "arn:aws:iam::867344451130:role/OSIJiraPipeline" + region: "us-east-2" + serverless: true +create-action: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + index: "no-more-plugin-setting" + index_type: "trace-analytics-raw" + action: "update" + aws: + sts_role_arn: "arn:aws:iam::867344451130:role/OSIJiraPipeline" + region: "us-east-2" + serverless: true +create-actions-with-expression: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + index: "no-more-plugin-setting" + index_type: "trace-analytics-raw" + actions: + - type: "create" + when: "${getMetadata(\"action\")}" + aws: + sts_role_arn: "arn:aws:iam::867344451130:role/OSIJiraPipeline" + region: "us-east-2" + serverless: true +invalid-max-retries: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + dlq: null + dlq_file: null + max_retries: -1 +no-dlq-file-path: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + dlq: null + dlq_file: null + max_retries: null +dlq-file-path-10-retries: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + dlq: null + dlq_file: foo.txt + max_retries: 10 +with-dlq-plugin-10-retries: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + dlq: + s3: + bucket: "my-dlq-bucket" + key_path_prefix: "dlq-files/" + dlq_file: null + max_retries: 10 +dlq-file-and-dlq-plugin: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + dlq: + s3: + bucket: "my-dlq-bucket" + key_path_prefix: "dlq-files/" + dlq_file: foo.txt + max_retries: 10 \ No newline at end of file diff --git a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/create-action-config.yaml b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/create-action-config.yaml deleted file mode 100644 index ec048e6d93..0000000000 --- a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/create-action-config.yaml +++ /dev/null @@ -1,10 +0,0 @@ -sink: - opensearch: - hosts: [ "http://localhost:9200" ] - index: "no-more-plugin-setting" - index_type: "trace-analytics-raw" - action: "update" - aws: - sts_role_arn: "arn:aws:iam::867344451130:role/OSIJiraPipeline" - region: "us-east-2" - serverless: true \ No newline at end of file diff --git a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/create-actions-with-expression-config.yaml b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/create-actions-with-expression-config.yaml deleted file mode 100644 index 7cd109e734..0000000000 --- a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/create-actions-with-expression-config.yaml +++ /dev/null @@ -1,12 +0,0 @@ -sink: - opensearch: - hosts: [ "http://localhost:9200" ] - index: "no-more-plugin-setting" - index_type: "trace-analytics-raw" - actions: - - type: "create" - when: "${getMetadata(\"action\")}" - aws: - sts_role_arn: "arn:aws:iam::867344451130:role/OSIJiraPipeline" - region: "us-east-2" - serverless: true \ No newline at end of file diff --git a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/dlq-file-path-10-retries-config.yaml b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/dlq-file-path-10-retries-config.yaml deleted file mode 100644 index e1e1bfa2bd..0000000000 --- a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/dlq-file-path-10-retries-config.yaml +++ /dev/null @@ -1,6 +0,0 @@ -sink: - opensearch: - hosts: [ "http://localhost:9200" ] - dlq: null - dlq_file: foo.txt - max_retries: 10 \ No newline at end of file diff --git a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-action-config.yaml b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-action-config.yaml deleted file mode 100644 index 800cc5c753..0000000000 --- a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-action-config.yaml +++ /dev/null @@ -1,10 +0,0 @@ -sink: - opensearch: - hosts: [ "http://localhost:9200" ] - index: "no-more-plugin-setting" - index_type: "trace-analytics-raw" - action: "invalid" - aws: - sts_role_arn: "arn:aws:iam::867344451130:role/OSIJiraPipeline" - region: "us-east-2" - serverless: true \ No newline at end of file diff --git a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-action-with-expression-config.yaml b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-action-with-expression-config.yaml deleted file mode 100644 index 6127f473c8..0000000000 --- a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-action-with-expression-config.yaml +++ /dev/null @@ -1,10 +0,0 @@ -sink: - opensearch: - hosts: [ "http://localhost:9200" ] - index: "no-more-plugin-setting" - index_type: "trace-analytics-raw" - action: "${anInvalidFunction()}" - aws: - sts_role_arn: "arn:aws:iam::867344451130:role/OSIJiraPipeline" - region: "us-east-2" - serverless: true \ No newline at end of file diff --git a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-actions-config.yaml b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-actions-config.yaml deleted file mode 100644 index 80ce9cf0b6..0000000000 --- a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-actions-config.yaml +++ /dev/null @@ -1,11 +0,0 @@ -sink: - opensearch: - hosts: [ "http://localhost:9200" ] - index: "no-more-plugin-setting" - index_type: "trace-analytics-raw" - actions: - - type: "invalid" - aws: - sts_role_arn: "arn:aws:iam::867344451130:role/OSIJiraPipeline" - region: "us-east-2" - serverless: true \ No newline at end of file diff --git a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-actions-with-expression-config.yaml b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-actions-with-expression-config.yaml deleted file mode 100644 index f94836cece..0000000000 --- a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-actions-with-expression-config.yaml +++ /dev/null @@ -1,11 +0,0 @@ -sink: - opensearch: - hosts: [ "http://localhost:9200" ] - index: "no-more-plugin-setting" - index_type: "trace-analytics-raw" - actions: - - type: "${anInvalidFunction()}" - aws: - sts_role_arn: "arn:aws:iam::867344451130:role/OSIJiraPipeline" - region: "us-east-2" - serverless: true \ No newline at end of file diff --git a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-max-retries-config.yaml b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-max-retries-config.yaml deleted file mode 100644 index 8452700433..0000000000 --- a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/invalid-max-retries-config.yaml +++ /dev/null @@ -1,6 +0,0 @@ -sink: - opensearch: - hosts: [ "http://localhost:9200" ] - dlq: null - dlq_file: null - max_retries: -1 \ No newline at end of file diff --git a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/no-dlq-file-path-config.yaml b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/no-dlq-file-path-config.yaml deleted file mode 100644 index 6e7bec4427..0000000000 --- a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/no-dlq-file-path-config.yaml +++ /dev/null @@ -1,6 +0,0 @@ -sink: - opensearch: - hosts: [ "http://localhost:9200" ] - dlq: null - dlq_file: null - max_retries: null \ No newline at end of file diff --git a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/valid_sink_config.yaml b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/valid_sink_config.yaml deleted file mode 100644 index ea8ebb564a..0000000000 --- a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/valid_sink_config.yaml +++ /dev/null @@ -1,9 +0,0 @@ -sink: - opensearch: - hosts: [ "http://localhost:9200" ] - index: "no-more-plugin-setting" - index_type: "trace-analytics-raw" - aws: - sts_role_arn: "arn:aws:iam::867344451130:role/OSIJiraPipeline" - region: "us-east-2" - serverless: true \ No newline at end of file diff --git a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/with-dlq-plugin-10-retries-config.yaml b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/with-dlq-plugin-10-retries-config.yaml deleted file mode 100644 index f8bd8f15b5..0000000000 --- a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/with-dlq-plugin-10-retries-config.yaml +++ /dev/null @@ -1,9 +0,0 @@ -sink: - opensearch: - hosts: [ "http://localhost:9200" ] - dlq: - s3: - bucket: "my-dlq-bucket" - key_path_prefix: "dlq-files/" - dlq_file: null - max_retries: 10 \ No newline at end of file From 915ce5c1f9fd70d01f176ea258e96ebec14aeb7c Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Mon, 16 Dec 2024 14:49:14 -0800 Subject: [PATCH 10/62] Rework ConnectionConfigurationTests.java Signed-off-by: Maxwell Brown --- .../sink/opensearch/RetryConfiguration.java | 1 - .../ConnectionConfigurationTests.java | 467 ++++++------------ .../open-search-sink-configurations.yaml | 193 +++++++- .../dlq-file-and-dlq-plugin-config.yaml | 9 - 4 files changed, 332 insertions(+), 338 deletions(-) delete mode 100644 data-prepper-plugins/opensearch/src/test/resources/test-configurations/dlq-file-and-dlq-plugin-config.yaml diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfiguration.java index f7269e3683..666a2dd543 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfiguration.java @@ -5,7 +5,6 @@ package org.opensearch.dataprepper.plugins.sink.opensearch; -import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.DlqConfiguration; import java.util.Optional; diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java index 382afeb869..dc68864a0b 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java @@ -5,6 +5,8 @@ package org.opensearch.dataprepper.plugins.sink.opensearch; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; @@ -17,20 +19,17 @@ import org.opensearch.client.transport.rest_client.RestClientTransport; import org.opensearch.dataprepper.aws.api.AwsCredentialsOptions; import org.opensearch.dataprepper.aws.api.AwsCredentialsSupplier; -import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.PreSerializedJsonpMapper; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.http.apache.ApacheHttpClient; import software.amazon.awssdk.regions.Region; +import java.io.File; import java.io.IOException; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; -import java.util.UUID; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; @@ -49,20 +48,45 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.opensearch.dataprepper.plugins.sink.opensearch.ConnectionConfiguration.SERVERLESS; -import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.DISTRIBUTION_VERSION; @ExtendWith(MockitoExtension.class) class ConnectionConfigurationTests { - private static final String PROXY_PARAMETER = "proxy"; + private static final String OPEN_SEARCH_SINK_CONFIGURATIONS = "open-search-sink-configurations.yaml"; + private static final String EMPTY_SINK_CONFIG = "empty-sink"; + private static final String ES6_DEFAULT_CONFIG = "es6-default"; + private static final String AWS_SERVERLESS_DEFAULT = "aws-serverless-default"; + private static final String AWS_SERVERLESS_NO_CERT = "aws-serverless-no-cert"; + private static final String BASIC_CREDENTIALS_NO_CERT = "basic-credentials-no-cert"; + private static final String BASIC_CREDENTIALS_NO_CERT_INSECURE = "basic-credentials-no-cert-insecure"; + private static final String BASIC_CREDENTIALS_WITH_CERT = "basic-credentials-with-cert"; + private static final String AWS_REGION_ONLY = "aws-region-only"; + private static final String SERVERLESS_OPTIONS = "serverless-options"; + private static final String AWS_REGION_ONLY_INSECURE = "aws-region-only-insecure"; + private static final String AWS_WITH_CERT = "aws-with-cert"; + private static final String AWS_WITH_CERT_AND_ARN = "aws-with-cert-and-arn"; + private static final String AWS_WITH_2_HEADER = "aws-with-2-header"; + private static final String VALID_PROXY_IP = "valid-proxy-ip"; + private static final String VALID_PROXY_HOST_NAME = "valid-proxy-host-name"; + private static final String VALID_HTTP_PROXY_SCHEME = "valid-http-proxy-scheme"; + private static final String INVALID_HTTP_PROXY_PORT = "invalid-http-proxy-port"; + private static final String INVALID_HTTP_PROXY_NO_PORT = "invalid-http-proxy-no-port"; + private static final String INVALID_HTTP_PROXY_PORT_NOT_IN_RANGE = "invalid-http-proxy-port-not-in-range"; + private static final String INVALID_HTTP_PROXY_NOT_HTTP = "invalid-http-proxy-not-http"; + private final List TEST_HOSTS = Collections.singletonList("http://localhost:9200"); - private final String TEST_USERNAME = "admin"; - private final String TEST_PASSWORD = "admin"; - private final String TEST_PIPELINE_NAME = "Test-Pipeline"; + private final String TEST_USERNAME = "test-username"; + private final String TEST_PASSWORD = "test-password"; private final Integer TEST_CONNECT_TIMEOUT = 5; private final Integer TEST_SOCKET_TIMEOUT = 10; - private final String TEST_CERT_PATH = Objects.requireNonNull(getClass().getClassLoader().getResource("test-ca.pem")).getFile(); private final String TEST_ROLE = "arn:aws:iam::123456789012:role/test-role"; + private final String TEST_NETWORK_POLICY = "test network policy"; + private final String TEST_COLLECTION_NAME = "test collection"; + private final String TEST_VPCE_ID = "test vpce id"; + private final String TEST_EXTERNAL_ID = "test-external-id"; + private final String TEST_HEADER_NAME_1 = "header1"; + private final String TEST_HEADER_NAME_2 = "header2"; + private final String TEST_HEADER_VALUE_1 = "test-header-1"; + private final String TEST_HEADER_VALUE_2 = "test-header-2"; @Mock private ApacheHttpClient.Builder apacheHttpClientBuilder; @@ -71,12 +95,13 @@ class ConnectionConfigurationTests { @Mock private AwsCredentialsSupplier awsCredentialsSupplier; + ObjectMapper objectMapper; + @Test - void testReadConnectionConfigurationDefault() { - final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, null, null, null, null, false, null, null, null, false); + void testReadConnectionConfigurationDefault() throws IOException { + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(EMPTY_SINK_CONFIG); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertEquals(TEST_HOSTS, connectionConfiguration.getHosts()); assertNull(connectionConfiguration.getUsername()); assertNull(connectionConfiguration.getPassword()); @@ -84,37 +109,30 @@ void testReadConnectionConfigurationDefault() { assertNull(connectionConfiguration.getCertPath()); assertNull(connectionConfiguration.getConnectTimeout()); assertNull(connectionConfiguration.getSocketTimeout()); - assertEquals(TEST_PIPELINE_NAME, connectionConfiguration.getPipelineName()); assertTrue(connectionConfiguration.isRequestCompressionEnabled()); } @Test - void testReadConnectionConfigurationES6Default() { - final Map configMetadata = generateConfigurationMetadata( - TEST_HOSTS, null, null, null, null, true, null, null, null, false); - configMetadata.put(DISTRIBUTION_VERSION, "es6"); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(configMetadata); + void testReadConnectionConfigurationES6Default() throws IOException { + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(ES6_DEFAULT_CONFIG); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertFalse(connectionConfiguration.isRequestCompressionEnabled()); } @Test - void testReadConnectionConfigurationAwsOptionServerlessDefault() { - final String testArn = TEST_ROLE; - final Map configMetadata = generateConfigurationMetadataWithAwsOption(TEST_HOSTS, null, null, null, null, true, false, null, testArn, null, TEST_CERT_PATH, false, Collections.emptyMap()); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(configMetadata); + void testReadConnectionConfigurationAwsOptionServerlessDefault() throws IOException { + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(AWS_SERVERLESS_DEFAULT); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertTrue(connectionConfiguration.isServerless()); } @Test void testCreateClientDefault() throws IOException { - final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, null, null, null, null, false, null, null, null, false); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(EMPTY_SINK_CONFIG); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); assertNotNull(client); client.close(); @@ -122,10 +140,9 @@ void testCreateClientDefault() throws IOException { @Test void testCreateOpenSearchClientDefault() throws IOException { - final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, null, null, null, null, false, null, null, null, false); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(EMPTY_SINK_CONFIG); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); final OpenSearchClient openSearchClient = connectionConfiguration.createOpenSearchClient(client, awsCredentialsSupplier); assertNotNull(openSearchClient); @@ -137,13 +154,9 @@ void testCreateOpenSearchClientDefault() throws IOException { @Test void testCreateOpenSearchClientAwsServerlessDefault() throws IOException { - final Map configMetadata = generateConfigurationMetadata( - TEST_HOSTS, null, null, null, null, true, null, null, null, false); - configMetadata.put(SERVERLESS, true); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(configMetadata); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(AWS_SERVERLESS_NO_CERT); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); - + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); final AwsCredentialsProvider awsCredentialsProvider = mock(AwsCredentialsProvider.class); when(awsCredentialsSupplier.getProvider(any())).thenReturn(awsCredentialsProvider); @@ -165,80 +178,35 @@ void testCreateOpenSearchClientAwsServerlessDefault() throws IOException { } @Test - void testReadConnectionConfigurationWithDeprecatedBasicCredentialsAndNoCert() { - final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, null, false); + void testReadConnectionConfigurationWithBasicCredentialsAndNoCert() throws IOException { + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(BASIC_CREDENTIALS_NO_CERT); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertEquals(TEST_HOSTS, connectionConfiguration.getHosts()); assertEquals(TEST_USERNAME, connectionConfiguration.getUsername()); assertEquals(TEST_PASSWORD, connectionConfiguration.getPassword()); assertEquals(TEST_CONNECT_TIMEOUT, connectionConfiguration.getConnectTimeout()); assertEquals(TEST_SOCKET_TIMEOUT, connectionConfiguration.getSocketTimeout()); assertFalse(connectionConfiguration.isAwsSigv4()); - assertEquals(TEST_PIPELINE_NAME, connectionConfiguration.getPipelineName()); } - @Test - void testReadConnectionConfigurationWithBasicCredentialsAndNoCert() { - final Map configurationMetadata = generateConfigurationMetadata( - TEST_HOSTS, null, null, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, null, false); - configurationMetadata.put("authentication", Map.of("username", TEST_USERNAME, "password", TEST_PASSWORD)); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(configurationMetadata); - final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); - assertEquals(TEST_HOSTS, connectionConfiguration.getHosts()); - assertNull(connectionConfiguration.getUsername()); - assertNull(connectionConfiguration.getPassword()); - assertNotNull(connectionConfiguration.getAuthConfig()); - assertEquals(TEST_USERNAME, connectionConfiguration.getAuthConfig().getUsername()); - assertEquals(TEST_PASSWORD, connectionConfiguration.getAuthConfig().getPassword()); - assertEquals(TEST_CONNECT_TIMEOUT, connectionConfiguration.getConnectTimeout()); - assertEquals(TEST_SOCKET_TIMEOUT, connectionConfiguration.getSocketTimeout()); - assertFalse(connectionConfiguration.isAwsSigv4()); - assertEquals(TEST_PIPELINE_NAME, connectionConfiguration.getPipelineName()); - } - - @Test - void testReadConnectionConfigurationWithBothDeprecatedBasicCredentialsAndAuthConfigShouldThrow() { - final Map configurationMetadata = generateConfigurationMetadata( - TEST_HOSTS, TEST_USERNAME, null, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, null, false); - configurationMetadata.put("authentication", Map.of("username", TEST_USERNAME, "password", TEST_PASSWORD)); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(configurationMetadata); - assertThrows(IllegalStateException.class, - () -> ConnectionConfiguration.readConnectionConfiguration(pluginSetting)); - } - - @Test - void testCreateClientWithDeprecatedBasicCredentialsAndNoCert() throws IOException { - final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, null, false); - final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); - final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); - assertNotNull(client); - client.close(); - } @Test void testCreateClientWithBasicCredentialsAndNoCert() throws IOException { - final Map configurationMetadata = generateConfigurationMetadata( - TEST_HOSTS, null, null, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, null, false); - configurationMetadata.put("authentication", Map.of("username", TEST_USERNAME, "password", TEST_PASSWORD)); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(configurationMetadata); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(BASIC_CREDENTIALS_NO_CERT); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); assertNotNull(client); client.close(); } + @Test void testCreateOpenSearchClientNoCert() throws IOException { - final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, null, false); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(BASIC_CREDENTIALS_NO_CERT); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); final OpenSearchClient openSearchClient = connectionConfiguration.createOpenSearchClient(client, awsCredentialsSupplier); assertNotNull(openSearchClient); @@ -249,10 +217,9 @@ void testCreateOpenSearchClientNoCert() throws IOException { @Test void testCreateClientInsecure() throws IOException { - final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, null, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(BASIC_CREDENTIALS_NO_CERT_INSECURE); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); assertNotNull(client); client.close(); @@ -260,10 +227,9 @@ void testCreateClientInsecure() throws IOException { @Test void testCreateOpenSearchClientInsecure() throws IOException { - final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, null, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(BASIC_CREDENTIALS_NO_CERT_INSECURE); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); final OpenSearchClient openSearchClient = connectionConfiguration.createOpenSearchClient(client, awsCredentialsSupplier); assertNotNull(openSearchClient); @@ -274,10 +240,9 @@ void testCreateOpenSearchClientInsecure() throws IOException { @Test void testCreateClientWithCertPath() throws IOException { - final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, false); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(BASIC_CREDENTIALS_WITH_CERT); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); assertNotNull(client); client.close(); @@ -285,10 +250,9 @@ void testCreateClientWithCertPath() throws IOException { @Test void testCreateOpenSearchClientWithCertPath() throws IOException { - final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, false); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(BASIC_CREDENTIALS_WITH_CERT); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); final OpenSearchClient openSearchClient = connectionConfiguration.createOpenSearchClient(client, awsCredentialsSupplier); assertNotNull(openSearchClient); @@ -299,88 +263,50 @@ void testCreateOpenSearchClientWithCertPath() throws IOException { @Test void testCreateClientWithAWSSigV4AndRegion() throws IOException { - final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, null, null, null, null, true, "us-west-2", null, null, false); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(AWS_REGION_ONLY); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); - assertEquals("us-west-2", connectionConfiguration.getAwsRegion()); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); + assertEquals("us-east-2", connectionConfiguration.getAwsRegion()); assertTrue(connectionConfiguration.isAwsSigv4()); } @Test void testServerlessOptions() throws IOException { - final String serverlessNetworkPolicyName = UUID.randomUUID().toString(); - final String serverlessCollectionName = UUID.randomUUID().toString(); - final String serverlessVpceId = UUID.randomUUID().toString(); - - final Map metadata = new HashMap<>(); - final Map awsOptionMetadata = new HashMap<>(); - final Map serverlessOptionsMetadata = new HashMap<>(); - serverlessOptionsMetadata.put("network_policy_name", serverlessNetworkPolicyName); - serverlessOptionsMetadata.put("collection_name", serverlessCollectionName); - serverlessOptionsMetadata.put("vpce_id", serverlessVpceId); - awsOptionMetadata.put("region", UUID.randomUUID().toString()); - awsOptionMetadata.put("serverless", true); - awsOptionMetadata.put("serverless_options", serverlessOptionsMetadata); - awsOptionMetadata.put("sts_role_arn", TEST_ROLE); - metadata.put("hosts", TEST_HOSTS); - metadata.put("username", UUID.randomUUID().toString()); - metadata.put("password", UUID.randomUUID().toString()); - metadata.put("connect_timeout", 1); - metadata.put("socket_timeout", 1); - metadata.put("aws", awsOptionMetadata); - - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(metadata); - final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(pluginSetting); - assertThat(connectionConfiguration.getServerlessNetworkPolicyName(), equalTo(serverlessNetworkPolicyName)); - assertThat(connectionConfiguration.getServerlessCollectionName(), equalTo(serverlessCollectionName)); - assertThat(connectionConfiguration.getServerlessVpceId(), equalTo(serverlessVpceId)); - } - - @Test - void testCreateClientWithAWSSigV4DefaultRegion() throws IOException { - final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, null, null, null, null, true, null, null, null, false); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(SERVERLESS_OPTIONS); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); - assertEquals("us-east-1", connectionConfiguration.getAwsRegion()); - assertTrue(connectionConfiguration.isAwsSigv4()); - assertEquals(TEST_PIPELINE_NAME, connectionConfiguration.getPipelineName()); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); + assertThat(connectionConfiguration.getServerlessNetworkPolicyName(), equalTo(TEST_NETWORK_POLICY)); + assertThat(connectionConfiguration.getServerlessCollectionName(), equalTo(TEST_COLLECTION_NAME)); + assertThat(connectionConfiguration.getServerlessVpceId(), equalTo(TEST_VPCE_ID)); } @Test void testCreateClientWithAWSSigV4AndInsecure() throws IOException { - final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, null, null, null, null, true, null, null, null, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(AWS_REGION_ONLY_INSECURE); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertEquals("us-east-1", connectionConfiguration.getAwsRegion()); assertTrue(connectionConfiguration.isAwsSigv4()); - assertEquals(TEST_PIPELINE_NAME, connectionConfiguration.getPipelineName()); } @Test void testCreateClientWithAWSSigV4AndCertPath() throws IOException { - final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, null, null, null, null, true, null, null, TEST_CERT_PATH, false); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(AWS_WITH_CERT); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertEquals("us-east-1", connectionConfiguration.getAwsRegion()); assertTrue(connectionConfiguration.isAwsSigv4()); - assertEquals(TEST_PIPELINE_NAME, connectionConfiguration.getPipelineName()); } @Test - void testCreateClientWithAWSSigV4AndSTSRole() { - final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, null, null, null, null, true, null, TEST_ROLE, TEST_CERT_PATH, false); + void testCreateClientWithAWSSigV4AndSTSRole() throws IOException { + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(AWS_WITH_CERT_AND_ARN); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertThat(connectionConfiguration, notNullValue()); assertThat(connectionConfiguration.getAwsRegion(), equalTo("us-east-1")); assertThat(connectionConfiguration.isAwsSigv4(), equalTo(true)); assertThat(connectionConfiguration.getAwsStsRoleArn(), equalTo(TEST_ROLE)); - assertThat(connectionConfiguration.getPipelineName(), equalTo(TEST_PIPELINE_NAME)); final AwsCredentialsProvider awsCredentialsProvider = mock(AwsCredentialsProvider.class); when(awsCredentialsSupplier.getProvider(any())).thenReturn(awsCredentialsProvider); @@ -397,16 +323,10 @@ void testCreateClientWithAWSSigV4AndSTSRole() { } @Test - void testCreateOpenSearchClientWithAWSSigV4AndSTSRole() { - final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, null, null, null, null, true, null, TEST_ROLE, TEST_CERT_PATH, false); + void testCreateOpenSearchClientWithAWSSigV4AndSTSRole() throws IOException { + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(AWS_WITH_CERT_AND_ARN); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); - assertThat(connectionConfiguration, notNullValue()); - assertThat(connectionConfiguration.getAwsRegion(), equalTo("us-east-1")); - assertThat(connectionConfiguration.isAwsSigv4(), equalTo(true)); - assertThat(connectionConfiguration.getAwsStsRoleArn(), equalTo(TEST_ROLE)); - assertThat(connectionConfiguration.getPipelineName(), equalTo(TEST_PIPELINE_NAME)); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); final AwsCredentialsProvider awsCredentialsProvider = mock(AwsCredentialsProvider.class); when(awsCredentialsSupplier.getProvider(any())).thenReturn(awsCredentialsProvider); @@ -430,17 +350,10 @@ void testCreateOpenSearchClientWithAWSSigV4AndSTSRole() { } @Test - void testCreateClientWithAWSOption() { - final String headerName1 = UUID.randomUUID().toString(); - final String headerValue1 = UUID.randomUUID().toString(); - final String headerName2 = UUID.randomUUID().toString(); - final String headerValue2 = UUID.randomUUID().toString(); - final String testArn = TEST_ROLE; - final String externalId = UUID.randomUUID().toString(); - final Map configurationMetadata = generateConfigurationMetadataWithAwsOption(TEST_HOSTS, null, null, null, null, false, true, null, testArn, externalId, null,false, Map.of(headerName1, headerValue1, headerName2, headerValue2)); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(configurationMetadata); + void testCreateClientWithAWSOption() throws IOException { + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(AWS_WITH_2_HEADER); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertThat(connectionConfiguration, notNullValue()); assertThat(connectionConfiguration.isAwsSigv4(), equalTo(true)); @@ -456,27 +369,20 @@ void testCreateClientWithAWSOption() { final AwsCredentialsOptions actualOptions = awsCredentialsOptionsArgumentCaptor.getValue(); assertThat(actualOptions.getStsRoleArn(), equalTo(TEST_ROLE)); - assertThat(actualOptions.getStsExternalId(), equalTo(externalId)); + assertThat(actualOptions.getStsExternalId(), equalTo(TEST_EXTERNAL_ID)); assertThat(actualOptions.getStsHeaderOverrides(), notNullValue()); assertThat(actualOptions.getStsHeaderOverrides().size(), equalTo(2)); - assertThat(actualOptions.getStsHeaderOverrides(), hasKey(headerName1)); - assertThat(actualOptions.getStsHeaderOverrides().get(headerName1), equalTo(headerValue1)); - assertThat(actualOptions.getStsHeaderOverrides(), hasKey(headerName2)); - assertThat(actualOptions.getStsHeaderOverrides().get(headerName2), equalTo(headerValue2)); + assertThat(actualOptions.getStsHeaderOverrides(), hasKey(TEST_HEADER_NAME_2)); + assertThat(actualOptions.getStsHeaderOverrides().get(TEST_HEADER_NAME_1), equalTo(TEST_HEADER_VALUE_1)); + assertThat(actualOptions.getStsHeaderOverrides(), hasKey(TEST_HEADER_NAME_2)); + assertThat(actualOptions.getStsHeaderOverrides().get(TEST_HEADER_NAME_2), equalTo(TEST_HEADER_VALUE_2)); } @Test - void testCreateOpenSearchClientWithAWSOption() { - final String headerName1 = UUID.randomUUID().toString(); - final String headerValue1 = UUID.randomUUID().toString(); - final String headerName2 = UUID.randomUUID().toString(); - final String headerValue2 = UUID.randomUUID().toString(); - final String testArn = TEST_ROLE; - final String externalId = UUID.randomUUID().toString(); - final Map configurationMetadata = generateConfigurationMetadataWithAwsOption(TEST_HOSTS, null, null, null, null, false, true, null, testArn, externalId, TEST_CERT_PATH, false, Map.of(headerName1, headerValue1, headerName2, headerValue2)); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(configurationMetadata); + void testCreateOpenSearchClientWithAWSOption() throws IOException { + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(AWS_WITH_2_HEADER); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertThat(connectionConfiguration, notNullValue()); assertThat(connectionConfiguration.isAwsSigv4(), equalTo(true)); @@ -489,7 +395,7 @@ void testCreateOpenSearchClientWithAWSOption() { final OpenSearchClient openSearchClient = connectionConfiguration.createOpenSearchClient(client, awsCredentialsSupplier); assertNotNull(openSearchClient); - assertThat(openSearchClient._transport(), instanceOf(AwsSdk2Transport.class)); + assertThat(openSearchClient._transport(), instanceOf(AwsSdk2Transport.class)); final AwsSdk2Transport opensearchTransport = (AwsSdk2Transport) openSearchClient._transport(); assertThat(opensearchTransport.options().credentials(), equalTo(awsCredentialsProvider)); @@ -500,23 +406,17 @@ void testCreateOpenSearchClientWithAWSOption() { assertThat(actualOptions.getStsRoleArn(), equalTo(TEST_ROLE)); assertThat(actualOptions.getStsHeaderOverrides(), notNullValue()); assertThat(actualOptions.getStsHeaderOverrides().size(), equalTo(2)); - assertThat(actualOptions.getStsHeaderOverrides(), hasKey(headerName1)); - assertThat(actualOptions.getStsHeaderOverrides().get(headerName1), equalTo(headerValue1)); - assertThat(actualOptions.getStsHeaderOverrides(), hasKey(headerName2)); - assertThat(actualOptions.getStsHeaderOverrides().get(headerName2), equalTo(headerValue2)); + assertThat(actualOptions.getStsHeaderOverrides(), hasKey(TEST_HEADER_NAME_2)); + assertThat(actualOptions.getStsHeaderOverrides().get(TEST_HEADER_NAME_1), equalTo(TEST_HEADER_VALUE_1)); + assertThat(actualOptions.getStsHeaderOverrides(), hasKey(TEST_HEADER_NAME_2)); + assertThat(actualOptions.getStsHeaderOverrides().get(TEST_HEADER_NAME_2), equalTo(TEST_HEADER_VALUE_2)); } @Test - void testCreateClientWithAWSSigV4AndHeaderOverrides() { - final String headerName1 = UUID.randomUUID().toString(); - final String headerValue1 = UUID.randomUUID().toString(); - final String headerName2 = UUID.randomUUID().toString(); - final String headerValue2 = UUID.randomUUID().toString(); - final Map configurationMetadata = generateConfigurationMetadata(TEST_HOSTS, null, null, null, null, true, null, TEST_ROLE, TEST_CERT_PATH, false); - configurationMetadata.put("aws_sts_header_overrides", Map.of(headerName1, headerValue1, headerName2, headerValue2)); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(configurationMetadata); + void testCreateClientWithAWSSigV4AndHeaderOverrides() throws IOException { + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(AWS_WITH_2_HEADER); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertThat(connectionConfiguration, notNullValue()); assertThat(connectionConfiguration.isAwsSigv4(), equalTo(true)); @@ -535,23 +435,17 @@ void testCreateClientWithAWSSigV4AndHeaderOverrides() { assertThat(actualOptions.getRegion(), equalTo(Region.US_EAST_1)); assertThat(actualOptions.getStsHeaderOverrides(), notNullValue()); assertThat(actualOptions.getStsHeaderOverrides().size(), equalTo(2)); - assertThat(actualOptions.getStsHeaderOverrides(), hasKey(headerName1)); - assertThat(actualOptions.getStsHeaderOverrides().get(headerName1), equalTo(headerValue1)); - assertThat(actualOptions.getStsHeaderOverrides(), hasKey(headerName2)); - assertThat(actualOptions.getStsHeaderOverrides().get(headerName2), equalTo(headerValue2)); + assertThat(actualOptions.getStsHeaderOverrides(), hasKey(TEST_HEADER_NAME_2)); + assertThat(actualOptions.getStsHeaderOverrides().get(TEST_HEADER_NAME_1), equalTo(TEST_HEADER_VALUE_1)); + assertThat(actualOptions.getStsHeaderOverrides(), hasKey(TEST_HEADER_NAME_2)); + assertThat(actualOptions.getStsHeaderOverrides().get(TEST_HEADER_NAME_2), equalTo(TEST_HEADER_VALUE_2)); } @Test - void testCreateOpenSearchClientWithAWSSigV4AndHeaderOverrides() { - final String headerName1 = UUID.randomUUID().toString(); - final String headerValue1 = UUID.randomUUID().toString(); - final String headerName2 = UUID.randomUUID().toString(); - final String headerValue2 = UUID.randomUUID().toString(); - final Map configurationMetadata = generateConfigurationMetadata(TEST_HOSTS, null, null, null, null, true, null, TEST_ROLE, TEST_CERT_PATH, false); - configurationMetadata.put("aws_sts_header_overrides", Map.of(headerName1, headerValue1, headerName2, headerValue2)); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(configurationMetadata); + void testCreateOpenSearchClientWithAWSSigV4AndHeaderOverrides() throws IOException { + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(AWS_WITH_2_HEADER); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertThat(connectionConfiguration, notNullValue()); assertThat(connectionConfiguration.isAwsSigv4(), equalTo(true)); @@ -577,21 +471,18 @@ void testCreateOpenSearchClientWithAWSSigV4AndHeaderOverrides() { assertThat(actualOptions.getRegion(), equalTo(Region.US_EAST_1)); assertThat(actualOptions.getStsHeaderOverrides(), notNullValue()); assertThat(actualOptions.getStsHeaderOverrides().size(), equalTo(2)); - assertThat(actualOptions.getStsHeaderOverrides(), hasKey(headerName1)); - assertThat(actualOptions.getStsHeaderOverrides().get(headerName1), equalTo(headerValue1)); - assertThat(actualOptions.getStsHeaderOverrides(), hasKey(headerName2)); - assertThat(actualOptions.getStsHeaderOverrides().get(headerName2), equalTo(headerValue2)); + assertThat(actualOptions.getStsHeaderOverrides(), hasKey(TEST_HEADER_NAME_2)); + assertThat(actualOptions.getStsHeaderOverrides().get(TEST_HEADER_NAME_1), equalTo(TEST_HEADER_VALUE_1)); + assertThat(actualOptions.getStsHeaderOverrides(), hasKey(TEST_HEADER_NAME_2)); + assertThat(actualOptions.getStsHeaderOverrides().get(TEST_HEADER_NAME_2), equalTo(TEST_HEADER_VALUE_2)); } @Test void testCreateAllClients_WithValidHttpProxy_HostIP() throws IOException { - final Map metadata = generateConfigurationMetadata( - TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, false); final String testHttpProxy = "121.121.121.121:80"; - metadata.put(PROXY_PARAMETER, testHttpProxy); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(metadata); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(VALID_PROXY_IP); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertEquals(connectionConfiguration.getProxy().get(), testHttpProxy); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); assertNotNull(client); @@ -601,13 +492,10 @@ void testCreateAllClients_WithValidHttpProxy_HostIP() throws IOException { @Test void testCreateAllClients_WithValidHttpProxy_HostName() throws IOException { - final Map metadata = generateConfigurationMetadata( - TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, false); final String testHttpProxy = "example.com:80"; - metadata.put(PROXY_PARAMETER, testHttpProxy); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(metadata); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(VALID_PROXY_HOST_NAME); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertEquals(connectionConfiguration.getProxy().get(), testHttpProxy); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); assertNotNull(client); @@ -617,13 +505,10 @@ void testCreateAllClients_WithValidHttpProxy_HostName() throws IOException { @Test void testCreateAllClients_WithValidHttpProxy_SchemeProvided() throws IOException { - final Map metadata = generateConfigurationMetadata( - TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, false); final String testHttpProxy = "http://example.com:4350"; - metadata.put(PROXY_PARAMETER, testHttpProxy); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(metadata); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(VALID_HTTP_PROXY_SCHEME); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertEquals(connectionConfiguration.getProxy().get(), testHttpProxy); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); assertNotNull(client); @@ -632,51 +517,39 @@ void testCreateAllClients_WithValidHttpProxy_SchemeProvided() throws IOException } @Test - void testCreateClient_WithInvalidHttpProxy_InvalidPort() { - final Map metadata = generateConfigurationMetadata( - TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, false); + void testCreateClient_WithInvalidHttpProxy_InvalidPort() throws IOException { final String testHttpProxy = "example.com:port"; - metadata.put(PROXY_PARAMETER, testHttpProxy); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(metadata); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(INVALID_HTTP_PROXY_PORT); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertEquals(connectionConfiguration.getProxy().get(), testHttpProxy); assertThrows(IllegalArgumentException.class, () -> connectionConfiguration.createClient(awsCredentialsSupplier)); } @Test - void testCreateClient_WithInvalidHttpProxy_NoPort() { - final Map metadata = generateConfigurationMetadata( - TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, false); + void testCreateClient_WithInvalidHttpProxy_NoPort() throws IOException { final String testHttpProxy = "example.com"; - metadata.put(PROXY_PARAMETER, testHttpProxy); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(metadata); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(INVALID_HTTP_PROXY_NO_PORT); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertThrows(IllegalArgumentException.class, () -> connectionConfiguration.createClient(awsCredentialsSupplier)); } @Test - void testCreateClient_WithInvalidHttpProxy_PortNotInRange() { - final Map metadata = generateConfigurationMetadata( - TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, false); + void testCreateClient_WithInvalidHttpProxy_PortNotInRange() throws IOException { final String testHttpProxy = "example.com:888888"; - metadata.put(PROXY_PARAMETER, testHttpProxy); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(metadata); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(INVALID_HTTP_PROXY_PORT_NOT_IN_RANGE); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertThrows(IllegalArgumentException.class, () -> connectionConfiguration.createClient(awsCredentialsSupplier)); } @Test - void testCreateClient_WithInvalidHttpProxy_NotHttp() { - final Map metadata = generateConfigurationMetadata( - TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, false); + void testCreateClient_WithInvalidHttpProxy_NotHttp() throws IOException { final String testHttpProxy = "socket://example.com:port"; - metadata.put(PROXY_PARAMETER, testHttpProxy); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(metadata); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(INVALID_HTTP_PROXY_NOT_HTTP); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertEquals(connectionConfiguration.getProxy().get(), testHttpProxy); assertThrows(IllegalArgumentException.class, () -> connectionConfiguration.createClient(awsCredentialsSupplier)); } @@ -691,62 +564,16 @@ void testCreateClient_WithConnectionConfigurationBuilder_ProxyOptionalObjectShou client.close(); } - private PluginSetting generatePluginSetting( - final List hosts, final String username, final String password, - final Integer connectTimeout, final Integer socketTimeout, final boolean awsSigv4, final String awsRegion, - final String awsStsRoleArn, final String certPath, final boolean insecure) { - final Map metadata = generateConfigurationMetadata(hosts, username, password, connectTimeout, socketTimeout, awsSigv4, awsRegion, awsStsRoleArn, certPath, insecure); - return getPluginSettingByConfigurationMetadata(metadata); - } - - private Map generateConfigurationMetadata( - final List hosts, final String username, final String password, - final Integer connectTimeout, final Integer socketTimeout, final boolean awsSigv4, final String awsRegion, - final String awsStsRoleArn, final String certPath, final boolean insecure) { - final Map metadata = new HashMap<>(); - metadata.put("hosts", hosts); - metadata.put("username", username); - metadata.put("password", password); - metadata.put("connect_timeout", connectTimeout); - metadata.put("socket_timeout", socketTimeout); - metadata.put("aws_sigv4", awsSigv4); - if (awsRegion != null) { - metadata.put("aws_region", awsRegion); - } - metadata.put("aws_sts_role_arn", awsStsRoleArn); - metadata.put("cert", certPath); - metadata.put("insecure", insecure); - return metadata; - } - - - private Map generateConfigurationMetadataWithAwsOption( - final List hosts, final String username, final String password, - final Integer connectTimeout, final Integer socketTimeout, final boolean serverless, final boolean awsSigv4, final String awsRegion, - final String awsStsRoleArn, final String awsStsExternalId, final String certPath, final boolean insecure, Map headerOverridesMap) { - final Map metadata = new HashMap<>(); - final Map awsOptionMetadata = new HashMap<>(); - metadata.put("hosts", hosts); - metadata.put("username", username); - metadata.put("password", password); - metadata.put("connect_timeout", connectTimeout); - metadata.put("socket_timeout", socketTimeout); - if (awsRegion != null) { - awsOptionMetadata.put("region", awsRegion); - } - awsOptionMetadata.put("serverless", serverless); - awsOptionMetadata.put("sts_role_arn", awsStsRoleArn); - awsOptionMetadata.put("sts_external_id", awsStsExternalId); - awsOptionMetadata.put("sts_header_overrides", headerOverridesMap); - metadata.put("aws", awsOptionMetadata); - metadata.put("cert", certPath); - metadata.put("insecure", insecure); - return metadata; - } - - private PluginSetting getPluginSettingByConfigurationMetadata(final Map metadata) { - final PluginSetting pluginSetting = new PluginSetting("opensearch", metadata); - pluginSetting.setPipelineName(TEST_PIPELINE_NAME); - return pluginSetting; - } -} + private OpenSearchSinkConfig generateOpenSearchSinkConfig(String pipelineName) throws IOException { + final File configurationFile = new File(getClass().getClassLoader().getResource(OPEN_SEARCH_SINK_CONFIGURATIONS).getFile()); + objectMapper = new ObjectMapper(new YAMLFactory()); + final Map pipelineConfigs = objectMapper.readValue(configurationFile, Map.class); + final Map pipelineConfig = (Map) pipelineConfigs.get(pipelineName); + final Map sinkMap = (Map) pipelineConfig.get("sink"); + final Map opensearchSinkMap = (Map) sinkMap.get("opensearch"); + String json = objectMapper.writeValueAsString(opensearchSinkMap); + OpenSearchSinkConfig openSearchSinkConfig = objectMapper.readValue(json, OpenSearchSinkConfig.class); + + return openSearchSinkConfig; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/opensearch/src/test/resources/open-search-sink-configurations.yaml b/data-prepper-plugins/opensearch/src/test/resources/open-search-sink-configurations.yaml index 3b4b904eaf..c68caba687 100644 --- a/data-prepper-plugins/opensearch/src/test/resources/open-search-sink-configurations.yaml +++ b/data-prepper-plugins/opensearch/src/test/resources/open-search-sink-configurations.yaml @@ -5,7 +5,7 @@ valid-sink: index: "no-more-plugin-setting" index_type: "trace-analytics-raw" aws: - sts_role_arn: "arn:aws:iam::867344451130:role/OSIJiraPipeline" + sts_role_arn: "arn:aws:iam::123456789012:role/test-role" region: "us-east-2" serverless: true invalid-action: @@ -16,7 +16,7 @@ invalid-action: index_type: "trace-analytics-raw" action: "invalid" aws: - sts_role_arn: "arn:aws:iam::867344451130:role/OSIJiraPipeline" + sts_role_arn: "arn:aws:iam::123456789012:role/test-role" region: "us-east-2" serverless: true invalid-actions: @@ -28,7 +28,7 @@ invalid-actions: actions: - type: "invalid" aws: - sts_role_arn: "arn:aws:iam::867344451130:role/OSIJiraPipeline" + sts_role_arn: "arn:aws:iam::123456789012:role/test-role" region: "us-east-2" serverless: true invalid-action-with-expression: @@ -39,7 +39,7 @@ invalid-action-with-expression: index_type: "trace-analytics-raw" action: "${anInvalidFunction()}" aws: - sts_role_arn: "arn:aws:iam::867344451130:role/OSIJiraPipeline" + sts_role_arn: "arn:aws:iam::123456789012:role/test-role" region: "us-east-2" serverless: true invalid-actions-with-expression: @@ -51,7 +51,7 @@ invalid-actions-with-expression: actions: - type: "${anInvalidFunction()}" aws: - sts_role_arn: "arn:aws:iam::867344451130:role/OSIJiraPipeline" + sts_role_arn: "arn:aws:iam::123456789012:role/test-role" region: "us-east-2" serverless: true create-action: @@ -62,7 +62,7 @@ create-action: index_type: "trace-analytics-raw" action: "update" aws: - sts_role_arn: "arn:aws:iam::867344451130:role/OSIJiraPipeline" + sts_role_arn: "arn:aws:iam::123456789012:role/test-role" region: "us-east-2" serverless: true create-actions-with-expression: @@ -75,7 +75,7 @@ create-actions-with-expression: - type: "create" when: "${getMetadata(\"action\")}" aws: - sts_role_arn: "arn:aws:iam::867344451130:role/OSIJiraPipeline" + sts_role_arn: "arn:aws:iam::123456789012:role/test-role" region: "us-east-2" serverless: true invalid-max-retries: @@ -118,4 +118,181 @@ dlq-file-and-dlq-plugin: bucket: "my-dlq-bucket" key_path_prefix: "dlq-files/" dlq_file: foo.txt - max_retries: 10 \ No newline at end of file + max_retries: 10 +empty-sink: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] +aws-serverless-default: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + cert: "./src/test/resources/test-ca.pem" + aws: + serverless: true + region: "us-east-2" + sts_role_arn: "arn:aws:iam::123456789012:role/test-role" +es6-default: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + distribution_version: "es6" +aws-serverless-no-cert: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + aws: + serverless: true + region: "us-east-2" + sts_role_arn: "arn:aws:iam::123456789012:role/test-role" +basic-credentials-no-cert: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + username: "test-username" + password: "test-password" + connect_timeout: 5 + socket_timeout: 10 +basic-credentials-no-cert-insecure: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + username: "test-username" + password: "test-password" + connect_timeout: 5 + socket_timeout: 10 + insecure: true +basic-credentials-with-cert: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + username: "test-username" + password: "test-password" + cert: "./src/test/resources/test-ca.pem" + connect_timeout: 5 + socket_timeout: 10 +aws-region-only: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + aws: + region: "us-east-2" +serverless-options: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + username: "test-username" + password: "test-password" + connect_timeout: 5 + socket_timeout: 10 + aws: + serverless: true + region: "us-east-2" + sts_role_arn: "arn:aws:iam::123456789012:role/test-role" + serverless_options: + network_policy_name: "test network policy" + collection_name: "test collection" + vpce_id: "test vpce id" +aws-region-only-insecure: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + aws: + region: "us-east-1" + insecure: true +aws-with-cert: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + aws: + region: "us-east-1" + cert: "./src/test/resources/test-ca.pem" +aws-with-cert-and-arn: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + aws: + region: "us-east-1" + sts_role_arn: "arn:aws:iam::123456789012:role/test-role" + cert: "./src/test/resources/test-ca.pem" +aws-with-2-header: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + aws: + region: "us-east-1" + sts_role_arn: "arn:aws:iam::123456789012:role/test-role" + sts_external_id: "test-external-id" + sts_header_overrides: + header1: "test-header-1" + header2: "test-header-2" +valid-proxy-ip: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + username: "test-username" + password: "test-password" + connect_timeout: 5 + socket_timeout: 10 + cert: "./src/test/resources/test-ca.pem" + proxy: "121.121.121.121:80" +valid-proxy-host-name: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + username: "test-username" + password: "test-password" + connect_timeout: 5 + socket_timeout: 10 + cert: "./src/test/resources/test-ca.pem" + proxy: "example.com:80" +valid-http-proxy-scheme: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + username: "test-username" + password: "test-password" + connect_timeout: 5 + socket_timeout: 10 + cert: "./src/test/resources/test-ca.pem" + proxy: "http://example.com:4350" +invalid-http-proxy-port: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + username: "test-username" + password: "test-password" + connect_timeout: 5 + socket_timeout: 10 + cert: "./src/test/resources/test-ca.pem" + proxy: "example.com:port" +invalid-http-proxy-no-port: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + username: "test-username" + password: "test-password" + connect_timeout: 5 + socket_timeout: 10 + cert: "./src/test/resources/test-ca.pem" + proxy: "example.com" +invalid-http-proxy-port-not-in-range: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + username: "test-username" + password: "test-password" + connect_timeout: 5 + socket_timeout: 10 + cert: "./src/test/resources/test-ca.pem" + proxy: "example.com:888888" +invalid-http-proxy-not-http: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + username: "test-username" + password: "test-password" + connect_timeout: 5 + socket_timeout: 10 + cert: "./src/test/resources/test-ca.pem" + proxy: "socket://example.com:port" \ No newline at end of file diff --git a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/dlq-file-and-dlq-plugin-config.yaml b/data-prepper-plugins/opensearch/src/test/resources/test-configurations/dlq-file-and-dlq-plugin-config.yaml deleted file mode 100644 index 94444597e1..0000000000 --- a/data-prepper-plugins/opensearch/src/test/resources/test-configurations/dlq-file-and-dlq-plugin-config.yaml +++ /dev/null @@ -1,9 +0,0 @@ -sink: - opensearch: - hosts: [ "http://localhost:9200" ] - dlq: - s3: - bucket: "my-dlq-bucket" - key_path_prefix: "dlq-files/" - dlq_file: foo.txt - max_retries: 10 \ No newline at end of file From 595b7662cfd425e4d893f91bc292ef0d50084797 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Mon, 16 Dec 2024 15:31:15 -0800 Subject: [PATCH 11/62] Rework IndexConfigurationTests.java.java Signed-off-by: Maxwell Brown --- .../opensearch/ConnectionConfiguration.java | 7 - .../opensearch/index/IndexConfiguration.java | 9 -- .../OpenSearchClientRefresherTest.java | 14 +- .../index/IndexConfigurationTests.java | 150 ++++++++---------- 4 files changed, 75 insertions(+), 105 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java index 511fc87a29..aeb840ba4b 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java @@ -248,13 +248,6 @@ public static ConnectionConfiguration readConnectionConfiguration(final OpenSear return builder.build(); } - public static ConnectionConfiguration readConnectionConfiguration(final PluginSetting pluginSetting){ - //Must fix all calls to this function - final List hosts = (List) pluginSetting.getAttributeFromSettings(HOSTS); - ConnectionConfiguration.Builder builder = new ConnectionConfiguration.Builder(hosts); - return builder.build(); - } - public RestHighLevelClient createClient(AwsCredentialsSupplier awsCredentialsSupplier) { final HttpHost[] httpHosts = new HttpHost[hosts.size()]; diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java index 749d9361d1..1a55590967 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java @@ -269,15 +269,6 @@ public static IndexConfiguration readIndexConfig(final OpenSearchSinkConfig open return builder.build(); } - public static IndexConfiguration readIndexConfig(final PluginSetting pluginSetting) { - return readIndexConfig(pluginSetting, null); - } - - public static IndexConfiguration readIndexConfig(final PluginSetting pluginSetting, final ExpressionEvaluator expressionEvaluator) { - IndexConfiguration.Builder builder = new IndexConfiguration.Builder(); - return builder.build(); - } - public IndexType getIndexType() { return indexType; diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresherTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresherTest.java index e879d47396..34658ae4de 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresherTest.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresherTest.java @@ -75,7 +75,7 @@ void testGetAfterUpdateWithDeprecatedBasicAuthUnchanged() { verify(clientFunction, times(1)).apply(any()); when(connectionConfiguration.getUsername()).thenReturn(TEST_USERNAME); when(connectionConfiguration.getPassword()).thenReturn(TEST_PASSWORD); - final PluginSetting newConfig = mock(PluginSetting.class); + final OpenSearchSinkConfig newConfig = mock(OpenSearchSinkConfig.class); final ConnectionConfiguration newConnectionConfiguration = mock(ConnectionConfiguration.class); when(newConnectionConfiguration.getUsername()).thenReturn(TEST_USERNAME); when(newConnectionConfiguration.getPassword()).thenReturn(TEST_PASSWORD); @@ -97,7 +97,7 @@ void testGetAfterUpdateWithBasicAuthUnchanged() { when(connectionConfiguration.getAuthConfig()).thenReturn(authConfig); when(authConfig.getUsername()).thenReturn(TEST_USERNAME); when(authConfig.getPassword()).thenReturn(TEST_PASSWORD); - final PluginSetting newConfig = mock(PluginSetting.class); + final OpenSearchSinkConfig newConfig = mock(OpenSearchSinkConfig.class); final ConnectionConfiguration newConnectionConfiguration = mock(ConnectionConfiguration.class); final AuthConfig newAuthConfig = mock(AuthConfig.class); when(newConnectionConfiguration.getAuthConfig()).thenReturn(newAuthConfig); @@ -121,7 +121,7 @@ void testGetAfterUpdateWithDeprecatedUsernameChanged() { verify(clientFunction, times(1)).apply(any()); assertThat(objectUnderTest.get(), equalTo(openSearchClient)); when(connectionConfiguration.getUsername()).thenReturn(TEST_USERNAME); - final PluginSetting newConfig = mock(PluginSetting.class); + final OpenSearchSinkConfig newConfig = mock(OpenSearchSinkConfig.class); final ConnectionConfiguration newConnectionConfiguration = mock(ConnectionConfiguration.class); when(newConnectionConfiguration.getUsername()).thenReturn(TEST_USERNAME + "_changed"); final OpenSearchClient newClient = mock(OpenSearchClient.class); @@ -147,7 +147,7 @@ void testGetAfterUpdateWithUsernameChanged() { when(connectionConfiguration.getAuthConfig()).thenReturn(authConfig); when(authConfig.getUsername()).thenReturn(TEST_USERNAME); when(authConfig.getPassword()).thenReturn(TEST_PASSWORD); - final PluginSetting newConfig = mock(PluginSetting.class); + final OpenSearchSinkConfig newConfig = mock(OpenSearchSinkConfig.class); final ConnectionConfiguration newConnectionConfiguration = mock(ConnectionConfiguration.class); final AuthConfig newAuthConfig = mock(AuthConfig.class); when(newConnectionConfiguration.getAuthConfig()).thenReturn(newAuthConfig); @@ -174,7 +174,7 @@ void testGetAfterUpdateWithDeprecatedPasswordChanged() { assertThat(objectUnderTest.get(), equalTo(openSearchClient)); when(connectionConfiguration.getUsername()).thenReturn(TEST_USERNAME); when(connectionConfiguration.getPassword()).thenReturn(TEST_PASSWORD); - final PluginSetting newConfig = mock(PluginSetting.class); + final OpenSearchSinkConfig newConfig = mock(OpenSearchSinkConfig.class); final ConnectionConfiguration newConnectionConfiguration = mock(ConnectionConfiguration.class); when(newConnectionConfiguration.getUsername()).thenReturn(TEST_USERNAME); when(newConnectionConfiguration.getPassword()).thenReturn(TEST_PASSWORD + "_changed"); @@ -201,7 +201,7 @@ void testGetAfterUpdateWithPasswordChanged() { when(connectionConfiguration.getAuthConfig()).thenReturn(authConfig); when(authConfig.getUsername()).thenReturn(TEST_USERNAME); when(authConfig.getPassword()).thenReturn(TEST_PASSWORD); - final PluginSetting newConfig = mock(PluginSetting.class); + final OpenSearchSinkConfig newConfig = mock(OpenSearchSinkConfig.class); final ConnectionConfiguration newConnectionConfiguration = mock(ConnectionConfiguration.class); final AuthConfig newAuthConfig = mock(AuthConfig.class); when(newConnectionConfiguration.getAuthConfig()).thenReturn(newAuthConfig); @@ -230,7 +230,7 @@ void testGetAfterUpdateClientFailure() { assertThat(objectUnderTest.get(), equalTo(openSearchClient)); when(connectionConfiguration.getUsername()).thenReturn(TEST_USERNAME); when(connectionConfiguration.getPassword()).thenReturn(TEST_PASSWORD); - final PluginSetting newConfig = mock(PluginSetting.class); + final OpenSearchSinkConfig newConfig = mock(OpenSearchSinkConfig.class); final ConnectionConfiguration newConnectionConfiguration = mock(ConnectionConfiguration.class); when(newConnectionConfiguration.getUsername()).thenReturn(TEST_USERNAME); when(newConnectionConfiguration.getPassword()).thenReturn(TEST_PASSWORD + "_changed"); diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfigurationTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfigurationTests.java index e14689e25d..481ff9eaa7 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfigurationTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfigurationTests.java @@ -17,6 +17,7 @@ import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException; import org.opensearch.dataprepper.plugins.sink.opensearch.DistributionVersion; +import org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchSinkConfig; import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.http.AbortableInputStream; import software.amazon.awssdk.services.s3.S3Client; @@ -68,6 +69,8 @@ public class IndexConfigurationTests { private static final String DEFAULT_TEMPLATE_FILE = "test-template-withshards.json"; private static final String TEST_CUSTOM_INDEX_POLICY_FILE = "test-custom-index-policy-file.json"; + ObjectMapper objectMapper; + @Test public void testRawAPMSpan() { final IndexConfiguration indexConfiguration = new IndexConfiguration.Builder().withIndexType( @@ -294,9 +297,9 @@ public void testValidCustomWithTemplateContent() throws JsonProcessingException @Test public void readIndexConfigWithTemplateFileAndTemplateContentUsesTemplateContent() throws JsonProcessingException { - final PluginSetting pluginSetting = generatePluginSetting("custom", "test", "test-file", createTemplateContent(), null, null, null); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig("custom", "test", "test-file", createTemplateContent(), null, null, null); - final IndexConfiguration objectUnderTest = IndexConfiguration.readIndexConfig(pluginSetting); + final IndexConfiguration objectUnderTest = IndexConfiguration.readIndexConfig(openSearchSinkConfig); assertThat(objectUnderTest, notNullValue()); assertThat(objectUnderTest.getIndexTemplate(), notNullValue()); @@ -304,12 +307,12 @@ public void readIndexConfigWithTemplateFileAndTemplateContentUsesTemplateContent } @Test - public void invalidTemplateContentThrowsInvalidPluginConfigurationException() { + public void invalidTemplateContentThrowsInvalidPluginConfigurationException() throws JsonProcessingException { final String invalidTemplateContent = UUID.randomUUID().toString(); - final PluginSetting pluginSetting = generatePluginSetting("custom", null, null, invalidTemplateContent, null, null, null); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig("custom", null, null, invalidTemplateContent, null, null, null); - assertThrows(InvalidPluginConfigurationException.class, () -> IndexConfiguration.readIndexConfig(pluginSetting)); + assertThrows(InvalidPluginConfigurationException.class, () -> IndexConfiguration.readIndexConfig(openSearchSinkConfig)); } @@ -322,11 +325,11 @@ public void testInvalidCustom() { } @Test - public void testReadIndexConfig_RawIndexType() { + public void testReadIndexConfig_RawIndexType() throws JsonProcessingException { final Map metadata = initializeConfigMetaData( IndexType.TRACE_ANALYTICS_RAW.getValue(), null, null, null, null, null, null); - final PluginSetting pluginSetting = getPluginSetting(metadata); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metadata); + final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig); final URL expTemplateFile = indexConfiguration .getClass().getClassLoader().getResource(RAW_DEFAULT_TEMPLATE_FILE); assertEquals(IndexType.TRACE_ANALYTICS_RAW, indexConfiguration.getIndexType()); @@ -340,19 +343,19 @@ public void testReadIndexConfig_RawIndexType() { } @Test - public void testReadIndexConfig_InvalidIndexTypeValueString() { + public void testReadIndexConfig_InvalidIndexTypeValueString() throws JsonProcessingException { final Map metadata = initializeConfigMetaData( "i-am-an-illegitimate-index-type", null, null, null, null, null, null); - final PluginSetting pluginSetting = getPluginSetting(metadata); - assertThrows(IllegalArgumentException.class, () -> IndexConfiguration.readIndexConfig(pluginSetting)); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metadata); + assertThrows(IllegalArgumentException.class, () -> IndexConfiguration.readIndexConfig(openSearchSinkConfig)); } @Test - public void testReadIndexConfig_ServiceMapIndexType() { + public void testReadIndexConfig_ServiceMapIndexType() throws JsonProcessingException { final Map metadata = initializeConfigMetaData( IndexType.TRACE_ANALYTICS_SERVICE_MAP.getValue(), null, null, null, null, null, null); - final PluginSetting pluginSetting = getPluginSetting(metadata); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metadata); + final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig); final URL expTemplateFile = indexConfiguration .getClass().getClassLoader().getResource(SERVICE_MAP_DEFAULT_TEMPLATE_FILE); assertEquals(IndexType.TRACE_ANALYTICS_SERVICE_MAP, indexConfiguration.getIndexType()); @@ -366,18 +369,19 @@ public void testReadIndexConfig_ServiceMapIndexType() { } @Test - public void testReadIndexConfigCustom() { + public void testReadIndexConfigCustom() throws JsonProcessingException { final String defaultTemplateFilePath = Objects.requireNonNull( getClass().getClassLoader().getResource(DEFAULT_TEMPLATE_FILE)).getFile(); final String testIndexAlias = "foo"; final long testBulkSize = 10L; final long testFlushTimeout = 30_000L; final String testIdField = "someId"; - final PluginSetting pluginSetting = generatePluginSetting( + final Map metaData = initializeConfigMetaData( null, testIndexAlias, defaultTemplateFilePath, null, testBulkSize, testFlushTimeout, testIdField); - pluginSetting.getSettings().put(IndexConfiguration.ESTIMATE_BULK_SIZE_USING_COMPRESSION, true); - pluginSetting.getSettings().put(IndexConfiguration.MAX_LOCAL_COMPRESSIONS_FOR_ESTIMATION, 5); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting); + metaData.put(IndexConfiguration.ESTIMATE_BULK_SIZE_USING_COMPRESSION, true); + metaData.put(IndexConfiguration.MAX_LOCAL_COMPRESSIONS_FOR_ESTIMATION, 5); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metaData); + final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig); assertEquals(IndexType.CUSTOM, indexConfiguration.getIndexType()); assertEquals(testIndexAlias, indexConfiguration.getIndexAlias()); assertFalse(indexConfiguration.getIndexTemplate().isEmpty()); @@ -389,7 +393,7 @@ public void testReadIndexConfigCustom() { } @Test - public void testReadIndexConfig_ExplicitCustomIndexType() { + public void testReadIndexConfig_ExplicitCustomIndexType() throws JsonProcessingException { final String defaultTemplateFilePath = Objects.requireNonNull( getClass().getClassLoader().getResource(DEFAULT_TEMPLATE_FILE)).getFile(); final String testIndexType = IndexType.CUSTOM.getValue(); @@ -399,8 +403,8 @@ public void testReadIndexConfig_ExplicitCustomIndexType() { final String testIdField = "someId"; final Map metadata = initializeConfigMetaData( testIndexType, testIndexAlias, defaultTemplateFilePath, null, testBulkSize, testFlushTimeout, testIdField); - final PluginSetting pluginSetting = getPluginSetting(metadata); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metadata); + final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig); assertEquals(IndexType.CUSTOM, indexConfiguration.getIndexType()); assertEquals(testIndexAlias, indexConfiguration.getIndexAlias()); assertFalse(indexConfiguration.getIndexTemplate().isEmpty()); @@ -410,27 +414,27 @@ public void testReadIndexConfig_ExplicitCustomIndexType() { } @Test - public void testReadIndexConfig_awsOptionServerlessDefault() { + public void testReadIndexConfig_awsOptionServerlessDefault() throws JsonProcessingException { final String testIndexAlias = "foo"; final Map metadata = initializeConfigMetaData( null, testIndexAlias, null, null, null, null, null); metadata.put(AWS_OPTION, Map.of(SERVERLESS, true)); metadata.put(TEMPLATE_TYPE, TemplateType.V1.getTypeName()); - final PluginSetting pluginSetting = getPluginSetting(metadata); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metadata); + final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig); assertEquals(IndexType.MANAGEMENT_DISABLED, indexConfiguration.getIndexType()); assertEquals(testIndexAlias, indexConfiguration.getIndexAlias()); } @Test - public void testReadIndexConfig_awsServerlessIndexTypeOverride() { + public void testReadIndexConfig_awsServerlessIndexTypeOverride() throws JsonProcessingException { final String testIndexAlias = "foo"; final Map metadata = initializeConfigMetaData( IndexType.CUSTOM.getValue(), testIndexAlias, null, null, null, null, null); metadata.put(AWS_OPTION, Map.of(SERVERLESS, true)); metadata.put(TEMPLATE_TYPE, TemplateType.V1.getTypeName()); - final PluginSetting pluginSetting = getPluginSetting(metadata); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metadata); + final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig); assertEquals(IndexType.CUSTOM, indexConfiguration.getIndexType()); assertEquals(testIndexAlias, indexConfiguration.getIndexAlias()); assertEquals(TemplateType.INDEX_TEMPLATE, indexConfiguration.getTemplateType()); @@ -438,83 +442,61 @@ public void testReadIndexConfig_awsServerlessIndexTypeOverride() { } @Test - public void testReadIndexConfig_distributionVersionDefault() { + public void testReadIndexConfig_distributionVersionDefault() throws JsonProcessingException { final Map metadata = initializeConfigMetaData( null, "foo", null,null, null, null, null); - final PluginSetting pluginSetting = getPluginSetting(metadata); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metadata); + final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig); assertEquals(indexConfiguration.getDistributionVersion(), DistributionVersion.DEFAULT); } @Test - public void testReadIndexConfig_es6Override() { + public void testReadIndexConfig_es6Override() throws JsonProcessingException { final Map metadata = initializeConfigMetaData( null, "foo", null, null, null, null, null); metadata.put(DISTRIBUTION_VERSION, "es6"); metadata.put(TEMPLATE_TYPE, TemplateType.INDEX_TEMPLATE.getTypeName()); - final PluginSetting pluginSetting = getPluginSetting(metadata); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metadata); + final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig); assertEquals(indexConfiguration.getDistributionVersion(), DistributionVersion.ES6); assertEquals(TemplateType.V1, indexConfiguration.getTemplateType()); assertEquals(IndexType.CUSTOM, indexConfiguration.getIndexType()); } @Test - public void testReadIndexConfig_documentRootKey() { + public void testReadIndexConfig_documentRootKey() throws JsonProcessingException { final Map metadata = initializeConfigMetaData( IndexType.CUSTOM.getValue(), "foo", null, null, null, null, null); final String expectedRootKey = UUID.randomUUID().toString(); metadata.put(DOCUMENT_ROOT_KEY, expectedRootKey); - final PluginSetting pluginSetting = getPluginSetting(metadata); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metadata); + final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig); assertEquals(expectedRootKey, indexConfiguration.getDocumentRootKey()); } @Test - public void testReadIndexConfig_emptyDocumentRootKey() { + public void testReadIndexConfig_emptyDocumentRootKey() throws JsonProcessingException { final Map metadata = initializeConfigMetaData( IndexType.CUSTOM.getValue(), "foo", null, null, null, null, null); metadata.put(DOCUMENT_ROOT_KEY, ""); - final PluginSetting pluginSetting = getPluginSetting(metadata); - assertThrows(IllegalArgumentException.class, () -> IndexConfiguration.readIndexConfig(pluginSetting)); - } - - @Test - public void testReadIndexConfig_pipeline() { - final Map metadata = initializeConfigMetaData( - IndexType.CUSTOM.getValue(), "foo", null, null, null, null, null); - final String expectedPipelineValue = UUID.randomUUID().toString(); - metadata.put(PIPELINE, expectedPipelineValue); - final PluginSetting pluginSetting = getPluginSetting(metadata); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting); - assertEquals(expectedPipelineValue, indexConfiguration.getPipeline()); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metadata); + assertThrows(IllegalArgumentException.class, () -> IndexConfiguration.readIndexConfig(openSearchSinkConfig)); } @Test - public void testReadIndexConfig_routing() { + public void testReadIndexConfig_routing() throws JsonProcessingException { final Map metadata = initializeConfigMetaData( IndexType.CUSTOM.getValue(), "foo", null, null, null, null, null); final String expectedRoutingValue = UUID.randomUUID().toString(); metadata.put(ROUTING, expectedRoutingValue); - final PluginSetting pluginSetting = getPluginSetting(metadata); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metadata); + final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig); assertEquals(expectedRoutingValue, indexConfiguration.getRouting()); } - @Test - public void testReadIndexConfig_routingField() { - final Map metadata = initializeConfigMetaData( - IndexType.CUSTOM.getValue(), "foo", null, null, null, null, null); - final String expectedRoutingFieldValue = UUID.randomUUID().toString(); - metadata.put(ROUTING_FIELD, expectedRoutingFieldValue); - final PluginSetting pluginSetting = getPluginSetting(metadata); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting); - assertEquals(expectedRoutingFieldValue, indexConfiguration.getRoutingField()); - } - @ParameterizedTest @ValueSource(strings = {"${key}", "${getMetadata(\"key\")}"}) - public void testReadIndexConfig_withValidDocumentVersionExpression(final String versionExpression) { + public void testReadIndexConfig_withValidDocumentVersionExpression(final String versionExpression) throws JsonProcessingException { final ExpressionEvaluator expressionEvaluator = mock(ExpressionEvaluator.class); when(expressionEvaluator.isValidFormatExpression(versionExpression)).thenReturn(true); @@ -523,16 +505,16 @@ public void testReadIndexConfig_withValidDocumentVersionExpression(final String IndexType.CUSTOM.getValue(), "foo", null, null, null, null, null); metadata.put(DOCUMENT_VERSION_EXPRESSION, versionExpression); - final PluginSetting pluginSetting = getPluginSetting(metadata); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metadata); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting, expressionEvaluator); + final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig, expressionEvaluator); assertThat(indexConfiguration, notNullValue()); assertThat(indexConfiguration.getVersionExpression(), equalTo(versionExpression)); } @Test - public void testReadIndexConfig_withInvalidDocumentVersionExpression_throws_InvalidPluginConfigurationException() { + public void testReadIndexConfig_withInvalidDocumentVersionExpression_throws_InvalidPluginConfigurationException() throws JsonProcessingException { final String versionExpression = UUID.randomUUID().toString(); final ExpressionEvaluator expressionEvaluator = mock(ExpressionEvaluator.class); @@ -542,40 +524,44 @@ public void testReadIndexConfig_withInvalidDocumentVersionExpression_throws_Inva IndexType.CUSTOM.getValue(), "foo", null, null, null, null, null); metadata.put(DOCUMENT_VERSION_EXPRESSION, versionExpression); - final PluginSetting pluginSetting = getPluginSetting(metadata); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metadata); - assertThrows(InvalidPluginConfigurationException.class, () -> IndexConfiguration.readIndexConfig(pluginSetting, expressionEvaluator)); + assertThrows(InvalidPluginConfigurationException.class, () -> IndexConfiguration.readIndexConfig(openSearchSinkConfig, expressionEvaluator)); } @Test - void getTemplateType_defaults_to_V1() { + void getTemplateType_defaults_to_V1() throws JsonProcessingException { final Map metadata = initializeConfigMetaData( IndexType.CUSTOM.getValue(), "foo", null, null, null, null, null); - final PluginSetting pluginSetting = getPluginSetting(metadata); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metadata); + final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig); assertThat(indexConfiguration.getTemplateType(), equalTo(TemplateType.V1)); } @ParameterizedTest @EnumSource(TemplateType.class) - void getTemplateType_with_configured_templateType(final TemplateType templateType) { + void getTemplateType_with_configured_templateType(final TemplateType templateType) throws JsonProcessingException { final Map metadata = initializeConfigMetaData( IndexType.CUSTOM.getValue(), "foo", null, null, null, null, null); metadata.put(TEMPLATE_TYPE, templateType.getTypeName()); - final PluginSetting pluginSetting = getPluginSetting(metadata); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metadata); + final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig); assertThat(indexConfiguration.getTemplateType(), equalTo(templateType)); } - private PluginSetting generatePluginSetting( + private OpenSearchSinkConfig generateOpenSearchSinkConfig( final String indexType, final String indexAlias, final String templateFilePath, final String templateContent, - final Long bulkSize, final Long flushTimeout, final String documentIdField) { + final Long bulkSize, final Long flushTimeout, final String documentIdField) throws JsonProcessingException { final Map metadata = initializeConfigMetaData(indexType, indexAlias, templateFilePath, templateContent, bulkSize, flushTimeout, documentIdField); - return getPluginSetting(metadata); + return getOpenSearchSinkConfig(metadata); } - private PluginSetting getPluginSetting(Map metadata) { - return new PluginSetting("opensearch", metadata); + private OpenSearchSinkConfig getOpenSearchSinkConfig(Map metadata) throws JsonProcessingException { + objectMapper = new ObjectMapper(); + String json = new ObjectMapper().writeValueAsString(metadata); + OpenSearchSinkConfig openSearchSinkConfig = objectMapper.readValue(json, OpenSearchSinkConfig.class); + + return openSearchSinkConfig; } private Map initializeConfigMetaData( From d150343edc6d67c5d614741ea11ec82712a9ff33 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Tue, 17 Dec 2024 15:17:33 -0800 Subject: [PATCH 12/62] copy files over from source Signed-off-by: Maxwell Brown --- .../opensearch/ConnectionConfiguration.java | 2 +- .../opensearch/OpenSearchClientRefresher.java | 2 +- .../sink/opensearch/OpenSearchSinkConfig.java | 2 +- .../AwsAuthenticationConfiguration.java | 63 +++++++++++++++++++ .../configuration/ServerlessOptions.java | 39 ++++++++++++ .../opensearch/index/IndexConfiguration.java | 2 +- 6 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AwsAuthenticationConfiguration.java create mode 100644 data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ServerlessOptions.java diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java index aeb840ba4b..5f84f24943 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java @@ -31,7 +31,7 @@ import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.PreSerializedJsonpMapper; import org.opensearch.dataprepper.plugins.source.opensearch.AuthConfig; -import org.opensearch.dataprepper.plugins.source.opensearch.configuration.AwsAuthenticationConfiguration; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.AwsAuthenticationConfiguration; import org.opensearch.dataprepper.plugins.source.opensearch.configuration.ServerlessOptions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresher.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresher.java index bee6242641..a12855aa93 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresher.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresher.java @@ -55,7 +55,7 @@ public OpenSearchClient get() { @Override public void update(final PluginSetting pluginSetting) { - return; + //not implemented } public void update(OpenSearchSinkConfig openSearchSinkConfig) { diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfig.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfig.java index b5a32f2d5c..785024a68a 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfig.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfig.java @@ -6,9 +6,9 @@ import org.apache.commons.lang3.EnumUtils; import org.opensearch.dataprepper.model.opensearch.OpenSearchBulkActions; import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.ActionConfiguration; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.AwsAuthenticationConfiguration; import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.DlqConfiguration; import org.opensearch.dataprepper.plugins.sink.opensearch.index.TemplateType; -import org.opensearch.dataprepper.plugins.source.opensearch.configuration.AwsAuthenticationConfiguration; import java.util.List; import java.util.Objects; diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AwsAuthenticationConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AwsAuthenticationConfiguration.java new file mode 100644 index 0000000000..b3d428b96f --- /dev/null +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AwsAuthenticationConfiguration.java @@ -0,0 +1,63 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.sink.opensearch.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.Size; +import org.opensearch.dataprepper.plugins.source.opensearch.configuration.ServerlessOptions; +import software.amazon.awssdk.regions.Region; + +import java.util.Map; + +public class AwsAuthenticationConfiguration { + + @JsonProperty("region") + @Size(min = 1, message = "Region cannot be empty string") + private String awsRegion; + + @JsonProperty("sts_role_arn") + @Size(min = 20, max = 2048, message = "awsStsRoleArn length should be between 1 and 2048 characters") + private String awsStsRoleArn; + + @JsonProperty("sts_external_id") + @Size(min = 2, max = 1224, message = "awsStsExternalId length should be between 2 and 1224 characters") + private String awsStsExternalId; + + @JsonProperty("sts_header_overrides") + @Size(max = 5, message = "sts_header_overrides supports a maximum of 5 headers to override") + private Map awsStsHeaderOverrides; + + @JsonProperty("serverless") + private Boolean serverless = false; + + @JsonProperty("serverless_options") + private ServerlessOptions serverlessOptions; + + public String getAwsStsRoleArn() { + return awsStsRoleArn; + } + + public String getAwsStsExternalId() { + return awsStsExternalId; + } + + public Region getAwsRegion() { + return awsRegion != null ? Region.of(awsRegion) : null; + } + + public Map getAwsStsHeaderOverrides() { + return awsStsHeaderOverrides; + } + + public Boolean isServerlessCollection() { + return serverless; + } + + public ServerlessOptions getServerlessOptions() { + return serverlessOptions; + } +} + diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ServerlessOptions.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ServerlessOptions.java new file mode 100644 index 0000000000..ddd3d0aa67 --- /dev/null +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ServerlessOptions.java @@ -0,0 +1,39 @@ +package org.opensearch.dataprepper.plugins.sink.opensearch.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class ServerlessOptions { + + @JsonProperty("network_policy_name") + private String networkPolicyName; + + @JsonProperty("collection_name") + private String collectionName; + + @JsonProperty("vpce_id") + private String vpceId; + + public ServerlessOptions() { + + } + + public ServerlessOptions(String networkPolicyName, String collectionName, String vpceId) { + this.networkPolicyName = networkPolicyName; + this.collectionName = collectionName; + this.vpceId = vpceId; + } + + public String getNetworkPolicyName() { + return networkPolicyName; + } + + public String getCollectionName() { + return collectionName; + } + + public String getVpceId() { + return vpceId; + } + +} + diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java index 1a55590967..6e4c675820 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java @@ -19,7 +19,7 @@ import org.opensearch.dataprepper.plugins.sink.opensearch.s3.FileReader; import org.opensearch.dataprepper.plugins.sink.opensearch.s3.S3ClientProvider; import org.opensearch.dataprepper.plugins.sink.opensearch.s3.S3FileReader; -import org.opensearch.dataprepper.plugins.source.opensearch.configuration.AwsAuthenticationConfiguration; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.AwsAuthenticationConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.arns.Arn; From 7b78c78379e184128c619a7868a538efe03b3672 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Tue, 17 Dec 2024 17:15:08 -0800 Subject: [PATCH 13/62] adjust all unit tests Signed-off-by: Maxwell Brown --- .../opensearch/ConnectionConfiguration.java | 3 +- .../opensearch/index/IndexConfiguration.java | 3 +- .../OpenSearchClientRefresherTest.java | 1 - .../sink/opensearch/OpenSearchSinkTest.java | 37 +++++++++++-------- .../index/IndexConfigurationTests.java | 3 -- 5 files changed, 24 insertions(+), 23 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java index 5f84f24943..5fdfb0a2ef 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java @@ -28,10 +28,9 @@ import org.opensearch.dataprepper.aws.api.AwsCredentialsOptions; import org.opensearch.dataprepper.aws.api.AwsCredentialsSupplier; import org.opensearch.dataprepper.aws.api.AwsRequestSigningApache4Interceptor; -import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.PreSerializedJsonpMapper; -import org.opensearch.dataprepper.plugins.source.opensearch.AuthConfig; import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.AwsAuthenticationConfiguration; +import org.opensearch.dataprepper.plugins.source.opensearch.AuthConfig; import org.opensearch.dataprepper.plugins.source.opensearch.configuration.ServerlessOptions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java index 6e4c675820..172f17c4b4 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java @@ -10,16 +10,15 @@ import org.apache.commons.lang3.EnumUtils; import org.opensearch.client.opensearch._types.VersionType; import org.opensearch.dataprepper.expression.ExpressionEvaluator; -import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.opensearch.OpenSearchBulkActions; import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException; import org.opensearch.dataprepper.plugins.sink.opensearch.DistributionVersion; import org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchSinkConfig; import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.ActionConfiguration; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.AwsAuthenticationConfiguration; import org.opensearch.dataprepper.plugins.sink.opensearch.s3.FileReader; import org.opensearch.dataprepper.plugins.sink.opensearch.s3.S3ClientProvider; import org.opensearch.dataprepper.plugins.sink.opensearch.s3.S3FileReader; -import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.AwsAuthenticationConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.arns.Arn; diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresherTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresherTest.java index 34658ae4de..9bbed3a456 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresherTest.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresherTest.java @@ -9,7 +9,6 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.client.opensearch.OpenSearchClient; import org.opensearch.dataprepper.metrics.PluginMetrics; -import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.plugins.source.opensearch.AuthConfig; import java.util.function.Function; diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java index 31b77e0bf3..05859535c3 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java @@ -22,6 +22,7 @@ import org.opensearch.dataprepper.expression.ExpressionEvaluator; import org.opensearch.dataprepper.metrics.MetricNames; import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.configuration.PipelineDescription; import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.event.EventHandle; @@ -41,10 +42,10 @@ import org.opensearch.dataprepper.plugins.sink.opensearch.index.TemplateType; import java.io.IOException; +import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.UUID; -import java.util.Collections; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; @@ -53,12 +54,12 @@ import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockConstruction; import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.mockito.Mockito.lenient; import static org.opensearch.dataprepper.model.sink.SinkLatencyMetrics.EXTERNAL_LATENCY; import static org.opensearch.dataprepper.model.sink.SinkLatencyMetrics.INTERNAL_LATENCY; import static org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchSink.BULKREQUEST_ERRORS; @@ -79,9 +80,6 @@ public class OpenSearchSinkTest { @Mock private OpenSearchClient openSearchClient; - @Mock - private PluginFactory pluginFactory; - @Mock private SinkContext sinkContext; @@ -97,6 +95,9 @@ public class OpenSearchSinkTest { @Mock private OpenSearchSinkConfiguration openSearchSinkConfiguration; + @Mock + private PipelineDescription pipelineDescription; + @Mock private IndexConfiguration indexConfiguration; @@ -109,6 +110,9 @@ public class OpenSearchSinkTest { @Mock private Timer bulkRequestTimer; + @Mock + private OpenSearchSinkConfig openSearchSinkConfig; + @Mock private Counter bulkRequestErrorsCounter; @@ -129,8 +133,7 @@ public class OpenSearchSinkTest { @BeforeEach void setup() { - when(pluginSetting.getPipelineName()).thenReturn(UUID.randomUUID().toString()); - when(pluginSetting.getName()).thenReturn(UUID.randomUUID().toString()); + when(pipelineDescription.getPipelineName()).thenReturn(UUID.randomUUID().toString()); final RetryConfiguration retryConfiguration = mock(RetryConfiguration.class); when(retryConfiguration.getDlq()).thenReturn(Optional.empty()); @@ -146,7 +149,6 @@ void setup() { when(indexConfiguration.getDocumentIdField()).thenReturn(null); when(indexConfiguration.getRoutingField()).thenReturn(null); when(indexConfiguration.getRouting()).thenReturn(null); - when(indexConfiguration.getPipeline()).thenReturn(null); when(indexConfiguration.getActions()).thenReturn(null); when(indexConfiguration.getDocumentRootKey()).thenReturn(null); lenient().when(indexConfiguration.getVersionType()).thenReturn(null); @@ -184,10 +186,10 @@ private OpenSearchSink createObjectUnderTest() throws IOException { indexManagerFactory = mock; })) { pluginMetricsMockedStatic.when(() -> PluginMetrics.fromPluginSetting(pluginSetting)).thenReturn(pluginMetrics); - openSearchSinkConfigurationMockedStatic.when(() -> OpenSearchSinkConfiguration.readESConfig(pluginSetting, expressionEvaluator)) + openSearchSinkConfigurationMockedStatic.when(() -> OpenSearchSinkConfiguration.readOSConfig(openSearchSinkConfig, expressionEvaluator)) .thenReturn(openSearchSinkConfiguration); return new OpenSearchSink( - pluginSetting, pluginFactory, sinkContext, expressionEvaluator, awsCredentialsSupplier, pluginConfigObservable); + pluginSetting, sinkContext, expressionEvaluator, awsCredentialsSupplier, pipelineDescription, pluginConfigObservable, openSearchSinkConfig); } } @@ -204,7 +206,7 @@ void test_initialization() throws IOException { @Test void doOutput_with_invalid_version_expression_catches_NumberFormatException_and_creates_DLQObject() throws IOException { - + when(pluginSetting.getName()).thenReturn("opensearch"); final String versionExpression = UUID.randomUUID().toString(); when(indexConfiguration.getVersionExpression()).thenReturn(versionExpression); @@ -214,6 +216,7 @@ void doOutput_with_invalid_version_expression_catches_NumberFormatException_and_ final EventHandle eventHandle = mock(EventHandle.class); when(event.getEventHandle()).thenReturn(eventHandle); final String index = UUID.randomUUID().toString(); + when(event.formatString(pipelineDescription.getPipelineName(), expressionEvaluator)).thenReturn(null); when(event.formatString(versionExpression, expressionEvaluator)).thenReturn("not_a_number"); when(event.formatString(indexConfiguration.getIndexAlias(), expressionEvaluator)).thenReturn(index); final Record eventRecord = new Record<>(event); @@ -234,7 +237,7 @@ void doOutput_with_invalid_version_expression_catches_NumberFormatException_and_ when(dlqObjectBuilder.withFailedData(failedDlqData.capture())).thenReturn(dlqObjectBuilder); when(dlqObjectBuilder.withPluginName(pluginSetting.getName())).thenReturn(dlqObjectBuilder); when(dlqObjectBuilder.withPluginId(pluginSetting.getName())).thenReturn(dlqObjectBuilder); - when(dlqObjectBuilder.withPipelineName(pluginSetting.getPipelineName())).thenReturn(dlqObjectBuilder); + when(dlqObjectBuilder.withPipelineName(pipelineDescription.getPipelineName())).thenReturn(dlqObjectBuilder); when(dlqObject.getFailedData()).thenReturn(mock(FailedDlqData.class)); doNothing().when(dlqObject).releaseEventHandle(false); @@ -294,14 +297,16 @@ void test_routing_in_document() throws IOException { void test_pipeline_in_document() throws IOException { String pipelineValue = UUID.randomUUID().toString(); String pipelineKey = UUID.randomUUID().toString(); + String pipelineName = UUID.randomUUID().toString(); + when(pipelineDescription.getPipelineName()).thenReturn(pipelineName); final OpenSearchSink objectUnderTest = createObjectUnderTest(); final Event event = JacksonEvent.builder() .withEventType("event") .withData(Collections.singletonMap(pipelineKey, pipelineValue)) .build(); - assertThat(objectUnderTest.getDocument(event).getPipelineField(), equalTo(Optional.empty())); + assertThat(objectUnderTest.getDocument(event).getPipelineField(), equalTo(Optional.of(pipelineName))); - when(indexConfiguration.getPipeline()).thenReturn("${"+pipelineKey+"}"); + when(pipelineDescription.getPipelineName()).thenReturn("${"+pipelineKey+"}"); final OpenSearchSink objectUnderTest2 = createObjectUnderTest(); assertThat(objectUnderTest2.getDocument(event).getPipelineField(), equalTo(Optional.of(pipelineValue))); } @@ -309,6 +314,7 @@ void test_pipeline_in_document() throws IOException { @Test void doOutput_with_invalid_version_expression_result_catches_RuntimeException_and_creates_DLQObject() throws IOException { + when(pluginSetting.getName()).thenReturn("opensearch"); final String versionExpression = UUID.randomUUID().toString(); when(indexConfiguration.getVersionExpression()).thenReturn(versionExpression); @@ -318,6 +324,7 @@ void doOutput_with_invalid_version_expression_result_catches_RuntimeException_an final EventHandle eventHandle = mock(EventHandle.class); when(event.getEventHandle()).thenReturn(eventHandle); final String index = UUID.randomUUID().toString(); + when(event.formatString(pipelineDescription.getPipelineName(), expressionEvaluator)).thenReturn(null); when(event.formatString(versionExpression, expressionEvaluator)).thenThrow(RuntimeException.class); when(event.formatString(indexConfiguration.getIndexAlias(), expressionEvaluator)).thenReturn(index); final Record eventRecord = new Record<>(event); @@ -338,7 +345,7 @@ void doOutput_with_invalid_version_expression_result_catches_RuntimeException_an when(dlqObjectBuilder.withFailedData(failedDlqData.capture())).thenReturn(dlqObjectBuilder); when(dlqObjectBuilder.withPluginName(pluginSetting.getName())).thenReturn(dlqObjectBuilder); when(dlqObjectBuilder.withPluginId(pluginSetting.getName())).thenReturn(dlqObjectBuilder); - when(dlqObjectBuilder.withPipelineName(pluginSetting.getPipelineName())).thenReturn(dlqObjectBuilder); + when(dlqObjectBuilder.withPipelineName(pipelineDescription.getPipelineName())).thenReturn(dlqObjectBuilder); when(dlqObject.getFailedData()).thenReturn(mock(FailedDlqData.class)); doNothing().when(dlqObject).releaseEventHandle(false); diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfigurationTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfigurationTests.java index 481ff9eaa7..3e61ba55c0 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfigurationTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfigurationTests.java @@ -14,7 +14,6 @@ import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.ValueSource; import org.opensearch.dataprepper.expression.ExpressionEvaluator; -import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException; import org.opensearch.dataprepper.plugins.sink.opensearch.DistributionVersion; import org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchSinkConfig; @@ -53,9 +52,7 @@ import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.DISTRIBUTION_VERSION; import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.DOCUMENT_ROOT_KEY; import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.DOCUMENT_VERSION_EXPRESSION; -import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.PIPELINE; import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.ROUTING; -import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.ROUTING_FIELD; import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.SERVERLESS; import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.TEMPLATE_TYPE; import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConstants.RAW_DEFAULT_TEMPLATE_FILE; From 55eb90b8cc072b690598d4c1d7abb05ec5cd2ae4 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Tue, 17 Dec 2024 17:22:46 -0800 Subject: [PATCH 14/62] spotless check Signed-off-by: Maxwell Brown --- .../dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java index 05859535c3..7d82abb916 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java @@ -29,7 +29,6 @@ import org.opensearch.dataprepper.model.event.JacksonEvent; import org.opensearch.dataprepper.model.failures.DlqObject; import org.opensearch.dataprepper.model.plugin.PluginConfigObservable; -import org.opensearch.dataprepper.model.plugin.PluginFactory; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.sink.SinkContext; import org.opensearch.dataprepper.plugins.sink.opensearch.dlq.FailedDlqData; From e5aad51e9a4c21a055478351eeadbf62749425e3 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Wed, 18 Dec 2024 13:37:55 -0800 Subject: [PATCH 15/62] Integration tests to OpenSearchSinkConfig from PluginSetting Signed-off-by: Maxwell Brown --- .../sink/opensearch/OpenSearchSinkIT.java | 233 ++++++++++-------- 1 file changed, 125 insertions(+), 108 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index 5b463c793f..ed5e8dfe03 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -38,6 +38,7 @@ import org.opensearch.dataprepper.expression.ExpressionEvaluator; import org.opensearch.dataprepper.metrics.MetricNames; import org.opensearch.dataprepper.metrics.MetricsTestUtil; +import org.opensearch.dataprepper.model.configuration.PipelineDescription; import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.event.EventType; @@ -129,6 +130,8 @@ public class OpenSearchSinkIT { private SinkContext sinkContext; private String testTagsTargetKey; + ObjectMapper objectMapper; + @Mock private PluginFactory pluginFactory; @@ -138,24 +141,33 @@ public class OpenSearchSinkIT { @Mock private ExpressionEvaluator expressionEvaluator; + @Mock + private OpenSearchSinkConfig openSearchSinkConfig; + + @Mock + private PipelineDescription pipelineDescription; + + @Mock + private PluginSetting pluginSetting; + @Mock private PluginConfigObservable pluginConfigObservable; - public OpenSearchSink createObjectUnderTest(PluginSetting pluginSetting, boolean doInitialize) { + public OpenSearchSink createObjectUnderTest(OpenSearchSinkConfig openSearchSinkConfig, boolean doInitialize) { OpenSearchSink sink = new OpenSearchSink( - pluginSetting, pluginFactory, null, expressionEvaluator, awsCredentialsSupplier, pluginConfigObservable); + pluginSetting, null, expressionEvaluator, awsCredentialsSupplier, pipelineDescription, pluginConfigObservable, openSearchSinkConfig); if (doInitialize) { sink.doInitialize(); } return sink; } - public OpenSearchSink createObjectUnderTestWithSinkContext(PluginSetting pluginSetting, boolean doInitialize) { + public OpenSearchSink createObjectUnderTestWithSinkContext(OpenSearchSinkConfig openSearchSinkConfig, boolean doInitialize) { sinkContext = mock(SinkContext.class); testTagsTargetKey = RandomStringUtils.randomAlphabetic(5); when(sinkContext.getTagsTargetKey()).thenReturn(testTagsTargetKey); OpenSearchSink sink = new OpenSearchSink( - pluginSetting, pluginFactory, sinkContext, expressionEvaluator, awsCredentialsSupplier, pluginConfigObservable); + pluginSetting, null, expressionEvaluator, awsCredentialsSupplier, pipelineDescription, pluginConfigObservable, openSearchSinkConfig); if (doInitialize) { sink.doInitialize(); } @@ -187,8 +199,8 @@ public void cleanOpenSearch() throws Exception { @Test @DisabledIf(value = "isES6", disabledReason = TRACE_INGESTION_TEST_DISABLED_REASON) public void testInstantiateSinkRawSpanDefault() throws IOException { - final PluginSetting pluginSetting = generatePluginSetting(IndexType.TRACE_ANALYTICS_RAW.getValue(), null, null); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(IndexType.TRACE_ANALYTICS_RAW.getValue(), null, null); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); final String indexAlias = IndexConstants.TYPE_TO_DEFAULT_ALIAS.get(IndexType.TRACE_ANALYTICS_RAW); assertThat(indexAlias, equalTo("otel-v1-apm-span")); Request request = new Request(HttpMethod.HEAD, indexAlias); @@ -215,7 +227,7 @@ public void testInstantiateSinkRawSpanDefault() throws IOException { assertThat(response.getStatusLine().getStatusCode(), equalTo(SC_OK)); // Instantiate sink again - sink = createObjectUnderTest(pluginSetting, true); + sink = createObjectUnderTest(openSearchSinkConfig, true); // Make sure no new write index *-000001 is created under alias final String rolloverIndexName = String.format("%s-000002", indexAlias); request = new Request(HttpMethod.GET, rolloverIndexName + "/_alias"); @@ -232,8 +244,8 @@ public void testInstantiateSinkRawSpanDefault() throws IOException { @Test @DisabledIf(value = "isES6", disabledReason = LOG_INGESTION_TEST_DISABLED_REASON) public void testInstantiateSinkLogsDefaultLogSink() throws IOException { - final PluginSetting pluginSetting = generatePluginSetting(IndexType.LOG_ANALYTICS.getValue(), null, null); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(IndexType.LOG_ANALYTICS.getValue(), null, null); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); final String indexAlias = IndexConstants.TYPE_TO_DEFAULT_ALIAS.get(IndexType.LOG_ANALYTICS); assertThat(indexAlias, equalTo("logs-otel-v1")); Request request = new Request(HttpMethod.HEAD, indexAlias); @@ -260,7 +272,7 @@ public void testInstantiateSinkLogsDefaultLogSink() throws IOException { assertThat(response.getStatusLine().getStatusCode(), equalTo(SC_OK)); // Instantiate sink again - sink = createObjectUnderTest(pluginSetting, true); + sink = createObjectUnderTest(openSearchSinkConfig, true); // Make sure no new write index *-000001 is created under alias final String rolloverIndexName = String.format("%s-000002", indexAlias); request = new Request(HttpMethod.GET, rolloverIndexName + "/_alias"); @@ -277,8 +289,8 @@ public void testInstantiateSinkLogsDefaultLogSink() throws IOException { @Test @DisabledIf(value = "isES6", disabledReason = METRIC_INGESTION_TEST_DISABLED_REASON) public void testInstantiateSinkMetricsDefaultMetricSink() throws IOException { - final PluginSetting pluginSetting = generatePluginSetting(IndexType.METRIC_ANALYTICS.getValue(), null, null); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(IndexType.METRIC_ANALYTICS.getValue(), null, null); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); final String indexAlias = IndexConstants.TYPE_TO_DEFAULT_ALIAS.get(IndexType.METRIC_ANALYTICS); assertThat(indexAlias, equalTo("metrics-otel-v1")); Request request = new Request(HttpMethod.HEAD, indexAlias); @@ -305,7 +317,7 @@ public void testInstantiateSinkMetricsDefaultMetricSink() throws IOException { assertThat(response.getStatusLine().getStatusCode(), equalTo(SC_OK)); // Instantiate sink again - sink = createObjectUnderTest(pluginSetting, true); + sink = createObjectUnderTest(openSearchSinkConfig, true); // Make sure no new write index *-000001 is created under alias final String rolloverIndexName = String.format("%s-000002", indexAlias); request = new Request(HttpMethod.GET, rolloverIndexName + "/_alias"); @@ -326,8 +338,8 @@ public void testInstantiateSinkRawSpanReservedAliasAlreadyUsedAsIndex() throws I final String reservedIndexAlias = IndexConstants.TYPE_TO_DEFAULT_ALIAS.get(IndexType.TRACE_ANALYTICS_RAW); final Request request = new Request(HttpMethod.PUT, reservedIndexAlias); client.performRequest(request); - final PluginSetting pluginSetting = generatePluginSetting(IndexType.TRACE_ANALYTICS_RAW.getValue(), null, null); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, false); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(IndexType.TRACE_ANALYTICS_RAW.getValue(), null, null); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, false); Assert.assertThrows(String.format(AbstractIndexManager.INDEX_ALIAS_USED_AS_INDEX_ERROR, reservedIndexAlias), RuntimeException.class, () -> sink.doInitialize()); } @@ -344,9 +356,9 @@ public void testOutputRawSpanDefault(final boolean estimateBulkSizeUsingCompress @SuppressWarnings("unchecked") final Map expData2 = mapper.readValue(testDoc2, Map.class); final List> testRecords = Arrays.asList(jsonStringToRecord(testDoc1), jsonStringToRecord(testDoc2)); - final PluginSetting pluginSetting = generatePluginSetting(IndexType.TRACE_ANALYTICS_RAW.getValue(), null, null, + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(IndexType.TRACE_ANALYTICS_RAW.getValue(), null, null, estimateBulkSizeUsingCompression, isRequestCompressionEnabled); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final String expIndexAlias = IndexConstants.TYPE_TO_DEFAULT_ALIAS.get(IndexType.TRACE_ANALYTICS_RAW); @@ -413,7 +425,7 @@ public void testOutputRawSpanWithDLQ(final boolean estimateBulkSizeUsingCompress @SuppressWarnings("unchecked") final Map expData = mapper.readValue(testDoc2, Map.class); final List> testRecords = Arrays.asList(jsonStringToRecord(testDoc1), jsonStringToRecord(testDoc2)); - final PluginSetting pluginSetting = generatePluginSetting(IndexType.TRACE_ANALYTICS_RAW.getValue(), null, null, + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(IndexType.TRACE_ANALYTICS_RAW.getValue(), null, null, estimateBulkSizeUsingCompression, isRequestCompressionEnabled); // generate temporary directory for dlq file final File tempDirectory = Files.createTempDirectory("").toFile(); @@ -421,7 +433,7 @@ public void testOutputRawSpanWithDLQ(final boolean estimateBulkSizeUsingCompress final String expDLQFile = tempDirectory.getAbsolutePath() + "/test-dlq.txt"; pluginSetting.getSettings().put(RetryConfiguration.DLQ_FILE, expDLQFile); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); sink.shutdown(); @@ -466,8 +478,8 @@ public void testOutputRawSpanWithDLQ(final boolean estimateBulkSizeUsingCompress @Test @DisabledIf(value = "isES6", disabledReason = TRACE_INGESTION_TEST_DISABLED_REASON) public void testInstantiateSinkServiceMapDefault() throws IOException { - final PluginSetting pluginSetting = generatePluginSetting(IndexType.TRACE_ANALYTICS_SERVICE_MAP.getValue(), null, null); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(IndexType.TRACE_ANALYTICS_SERVICE_MAP.getValue(), null, null); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); final String indexAlias = IndexConstants.TYPE_TO_DEFAULT_ALIAS.get(IndexType.TRACE_ANALYTICS_SERVICE_MAP); final Request request = new Request(HttpMethod.HEAD, indexAlias); final Response response = client.performRequest(request); @@ -492,9 +504,9 @@ public void testOutputServiceMapDefault(final boolean estimateBulkSizeUsingCompr @SuppressWarnings("unchecked") final Map expData = mapper.readValue(testDoc, Map.class); final List> testRecords = Collections.singletonList(jsonStringToRecord(testDoc)); - final PluginSetting pluginSetting = generatePluginSetting(IndexType.TRACE_ANALYTICS_SERVICE_MAP.getValue(), null, null, + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(IndexType.TRACE_ANALYTICS_SERVICE_MAP.getValue(), null, null, estimateBulkSizeUsingCompression, isRequestCompressionEnabled); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final String expIndexAlias = IndexConstants.TYPE_TO_DEFAULT_ALIAS.get(IndexType.TRACE_ANALYTICS_SERVICE_MAP); final List> retSources = getSearchResponseDocSources(expIndexAlias); @@ -524,7 +536,7 @@ public void testOutputServiceMapDefault(final boolean estimateBulkSizeUsingCompr assertThat(bulkRequestSizeBytesMetrics.get(2).getValue(), closeTo(expectedBulkRequestSizeBytes, 0)); // Check restart for index already exists - sink = createObjectUnderTest(pluginSetting, true); + sink = createObjectUnderTest(openSearchSinkConfig, true); sink.shutdown(); } @@ -533,8 +545,8 @@ public void testInstantiateSinkCustomIndex_NoRollOver() throws IOException { final String testIndexAlias = "test-alias"; final String testTemplateFile = Objects.requireNonNull( getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE)).getFile(); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, testTemplateFile); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); final String extraURI = DeclaredOpenSearchVersion.OPENDISTRO_0_10.compareTo( OpenSearchIntegrationHelper.getVersion()) >= 0 ? INCLUDE_TYPE_NAME_FALSE_URI : ""; final Request request = new Request(HttpMethod.HEAD, testIndexAlias + extraURI); @@ -543,7 +555,7 @@ public void testInstantiateSinkCustomIndex_NoRollOver() throws IOException { sink.shutdown(); // Check restart for index already exists - sink = createObjectUnderTest(pluginSetting, true); + sink = createObjectUnderTest(openSearchSinkConfig, true); sink.shutdown(); } @@ -559,9 +571,9 @@ public void testInstantiateSinkCustomIndex_WithIsmPolicy( final Map metadata = initializeConfigurationMetadata(null, indexAlias, testTemplateFile); metadata.put(IndexConfiguration.ISM_POLICY_FILE, TEST_CUSTOM_INDEX_POLICY_FILE); metadata.put(IndexConfiguration.TEMPLATE_TYPE, templateType); - final PluginSetting pluginSetting = generatePluginSettingByMetadata(metadata); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); final String extraURI = DeclaredOpenSearchVersion.OPENDISTRO_0_10.compareTo( OpenSearchIntegrationHelper.getVersion()) >= 0 ? INCLUDE_TYPE_NAME_FALSE_URI : ""; Request request = new Request(HttpMethod.HEAD, indexAlias + extraURI); @@ -605,7 +617,7 @@ public void testInstantiateSinkCustomIndex_WithIsmPolicy( assertThat(response.getStatusLine().getStatusCode(), equalTo(SC_OK)); // Instantiate sink again - sink = createObjectUnderTest(pluginSetting, true); + sink = createObjectUnderTest(openSearchSinkConfig, true); // Make sure no new write index *-000001 is created under alias final String rolloverIndexName = String.format("%s-000002", indexAlias); request = new Request(HttpMethod.GET, rolloverIndexName + "/_alias"); @@ -633,8 +645,8 @@ public void testInstantiateSinkDoesNotOverwriteNewerIndexTemplates( final String testTemplateFileV2 = getClass().getClassLoader().getResource(v2File).getFile(); // Create sink with template version 1 - PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, templateType, testTemplateFileV1); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, templateType, testTemplateFileV1); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); final String extraURI = DeclaredOpenSearchVersion.OPENDISTRO_0_10.compareTo( OpenSearchIntegrationHelper.getVersion()) >= 0 ? INCLUDE_TYPE_NAME_FALSE_URI : ""; @@ -652,8 +664,8 @@ public void testInstantiateSinkDoesNotOverwriteNewerIndexTemplates( sink.shutdown(); // Create sink with template version 2 - pluginSetting = generatePluginSetting(null, testIndexAlias, templateType, testTemplateFileV2); - sink = createObjectUnderTest(pluginSetting, true); + openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, templateType, testTemplateFileV2); + sink = createObjectUnderTest(openSearchSinkConfig, true); getTemplateRequest = new Request(HttpMethod.GET, "/" + templatePath + "/" + expectedIndexTemplateName + extraURI); @@ -669,8 +681,8 @@ public void testInstantiateSinkDoesNotOverwriteNewerIndexTemplates( sink.shutdown(); // Create sink with template version 1 again - pluginSetting = generatePluginSetting(null, testIndexAlias, templateType, testTemplateFileV1); - sink = createObjectUnderTest(pluginSetting, true); + openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, templateType, testTemplateFileV1); + sink = createObjectUnderTest(openSearchSinkConfig, true); getTemplateRequest = new Request(HttpMethod.GET, "/" + templatePath + "/" + expectedIndexTemplateName + extraURI); @@ -702,8 +714,8 @@ public void testIndexNameWithDateNotAsSuffixCreatesIndexTemplate( final String expectedIndexTemplateName = "prefix-suffix-index-template"; final String testTemplateFileV1 = getClass().getClassLoader().getResource(templateFile).getFile(); - PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, templateType, testTemplateFileV1); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, templateType, testTemplateFileV1); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); final String extraURI = DeclaredOpenSearchVersion.OPENDISTRO_0_10.compareTo( OpenSearchIntegrationHelper.getVersion()) >= 0 ? INCLUDE_TYPE_NAME_FALSE_URI : ""; @@ -814,9 +826,9 @@ public void testOutputCustomIndex() throws IOException, InterruptedException { final String testIdField = "someId"; final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, testTemplateFile); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List> retSources = getSearchResponseDocSources(testIndexAlias); assertThat(retSources.size(), equalTo(1)); @@ -840,10 +852,10 @@ public void testOpenSearchBulkActionsCreate() throws IOException, InterruptedExc final String testIdField = "someId"; final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, testTemplateFile); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); pluginSetting.getSettings().put(IndexConfiguration.ACTION, OpenSearchBulkActions.CREATE.toString()); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List> retSources = getSearchResponseDocSources(testIndexAlias); assertThat(retSources.size(), equalTo(1)); @@ -867,7 +879,7 @@ public void testOpenSearchBulkActionsCreateWithExpression() throws IOException, final String testIdField = "someId"; final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, testTemplateFile); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); Event event = (Event) testRecords.get(0).getData(); event.getMetadata().setAttribute("action", "create"); @@ -876,7 +888,7 @@ public void testOpenSearchBulkActionsCreateWithExpression() throws IOException, when(expressionEvaluator.isValidExpressionStatement("getMetadata(\"action\")")).thenReturn(true); when(expressionEvaluator.evaluate("getMetadata(\"action\")", event)).thenReturn(event.getMetadata().getAttribute("action")); pluginSetting.getSettings().put(IndexConfiguration.ACTION, actionFormatExpression); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List> retSources = getSearchResponseDocSources(testIndexAlias); assertThat(retSources.size(), equalTo(1)); @@ -900,7 +912,7 @@ public void testOpenSearchBulkActionsCreateWithInvalidExpression() throws IOExce final String testIdField = "someId"; final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, testTemplateFile); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); Event event = (Event) testRecords.get(0).getData(); event.getMetadata().setAttribute("action", "unknown"); @@ -909,7 +921,7 @@ public void testOpenSearchBulkActionsCreateWithInvalidExpression() throws IOExce when(expressionEvaluator.isValidExpressionStatement("getMetadata(\"action\")")).thenReturn(true); when(expressionEvaluator.evaluate("getMetadata(\"action\")", event)).thenReturn(event.getMetadata().getAttribute("action")); pluginSetting.getSettings().put(IndexConfiguration.ACTION, actionFormatExpression); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List> retSources = getSearchResponseDocSources(testIndexAlias); assertThat(retSources.size(), equalTo(0)); @@ -926,14 +938,14 @@ public void testBulkActionCreateWithActions() throws IOException, InterruptedExc final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, testTemplateFile); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.CREATE.toString()); aList.add(aMap); pluginSetting.getSettings().put(IndexConfiguration.ACTIONS, aList); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List> retSources = getSearchResponseDocSources(testIndexAlias); assertThat(retSources.size(), equalTo(1)); @@ -959,14 +971,14 @@ public void testBulkActionUpdateWithActions() throws IOException, InterruptedExc final String testId = "foo"; List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson2(testIdField, testId, "name", "value1"))); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, testTemplateFile); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.CREATE.toString()); aList.add(aMap); pluginSetting.getSettings().put(IndexConfiguration.ACTIONS, aList); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); List> retSources = getSearchResponseDocSources(testIndexAlias); assertThat(retSources.size(), equalTo(1)); @@ -986,7 +998,7 @@ public void testBulkActionUpdateWithActions() throws IOException, InterruptedExc aMap.put("type", OpenSearchBulkActions.UPDATE.toString()); aList.add(aMap); pluginSetting.getSettings().put(IndexConfiguration.ACTIONS, aList); - sink = createObjectUnderTest(pluginSetting, true); + sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); retSources = getSearchResponseDocSources(testIndexAlias); @@ -1015,7 +1027,7 @@ public void testBulkActionUpdateWithDocumentRootKey() throws IOException, Interr List> testRecords = Collections.singletonList(jsonStringToRecord(createJsonEvent)); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, testTemplateFile); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ROOT_KEY, documentRootKey); pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); @@ -1026,7 +1038,7 @@ public void testBulkActionUpdateWithDocumentRootKey() throws IOException, Interr aList.add(actionMap); pluginSetting.getSettings().put(IndexConfiguration.ACTIONS, aList); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); List> retSources = getSearchResponseDocSources(testIndexAlias); assertThat(retSources.size(), equalTo(1)); @@ -1051,7 +1063,7 @@ public void testBulkActionUpdateWithDocumentRootKey() throws IOException, Interr actionMap.put("type", OpenSearchBulkActions.UPDATE.toString()); aList.add(actionMap); pluginSetting.getSettings().put(IndexConfiguration.ACTIONS, aList); - sink = createObjectUnderTest(pluginSetting, true); + sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); retSources = getSearchResponseDocSources(testIndexAlias); @@ -1078,10 +1090,10 @@ public void testBulkActionUpsertWithActionsAndNoCreate() throws IOException, Int actionMap.put("type", OpenSearchBulkActions.UPSERT.toString()); aList.add(actionMap); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, testTemplateFile); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); pluginSetting.getSettings().put(IndexConfiguration.ACTIONS, aList); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); List> retSources = getSearchResponseDocSources(testIndexAlias); @@ -1104,14 +1116,14 @@ public void testBulkActionUpsertWithActions() throws IOException, InterruptedExc final String testId = "foo"; List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson2(testIdField, testId, "name", "value1"))); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, testTemplateFile); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.CREATE.toString()); aList.add(aMap); pluginSetting.getSettings().put(IndexConfiguration.ACTIONS, aList); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); List> retSources = getSearchResponseDocSources(testIndexAlias); assertThat(retSources.size(), equalTo(1)); @@ -1131,7 +1143,7 @@ public void testBulkActionUpsertWithActions() throws IOException, InterruptedExc aMap.put("type", OpenSearchBulkActions.UPSERT.toString()); aList.add(aMap); pluginSetting.getSettings().put(IndexConfiguration.ACTIONS, aList); - sink = createObjectUnderTest(pluginSetting, true); + sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); retSources = getSearchResponseDocSources(testIndexAlias); @@ -1152,14 +1164,14 @@ public void testBulkActionUpsertWithoutCreate() throws IOException, InterruptedE final String testIdField = "someId"; final String testId = "foo"; List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson3(testIdField, testId, "name", "value1", "newKey", "newValue"))); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, testTemplateFile); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.UPSERT.toString()); aList.add(aMap); pluginSetting.getSettings().put(IndexConfiguration.ACTIONS, aList); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); List> retSources = getSearchResponseDocSources(testIndexAlias); @@ -1188,14 +1200,14 @@ public void testBulkActionDeleteWithActions() throws IOException, InterruptedExc final String testId = "foo"; List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, testTemplateFile); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.DELETE.toString()); aList.add(aMap); pluginSetting.getSettings().put(IndexConfiguration.ACTIONS, aList); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); List> retSources = getSearchResponseDocSources(testIndexAlias); assertThat(retSources.size(), equalTo(0)); @@ -1214,8 +1226,8 @@ public void testEventOutputWithTags() throws IOException, InterruptedException { final List> testRecords = Collections.singletonList(new Record<>(testEvent)); - final PluginSetting pluginSetting = generatePluginSetting(IndexType.TRACE_ANALYTICS_RAW.getValue(), null, null); - final OpenSearchSink sink = createObjectUnderTestWithSinkContext(pluginSetting, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(IndexType.TRACE_ANALYTICS_RAW.getValue(), null, null); + final OpenSearchSink sink = createObjectUnderTestWithSinkContext(openSearchSinkConfig, true); sink.output(testRecords); final String expIndexAlias = IndexConstants.TYPE_TO_DEFAULT_ALIAS.get(IndexType.TRACE_ANALYTICS_RAW); @@ -1241,8 +1253,8 @@ public void testEventOutput() throws IOException, InterruptedException { final List> testRecords = Collections.singletonList(new Record<>(testEvent)); - final PluginSetting pluginSetting = generatePluginSetting(IndexType.TRACE_ANALYTICS_RAW.getValue(), null, null); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(IndexType.TRACE_ANALYTICS_RAW.getValue(), null, null); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); verify(pluginConfigObservable).addPluginConfigObserver(any()); sink.output(testRecords); @@ -1272,8 +1284,8 @@ public void testEventOutputWithSpecialAndExtremeValues(final Object testValue) t final List> testRecords = Collections.singletonList(new Record<>(testEvent)); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, null); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, null); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List> retSources = getSearchResponseDocSources(testIndexAlias); @@ -1298,9 +1310,9 @@ public void testOpenSearchDocumentId(final String testDocumentIdField) throws IO final List> testRecords = Collections.singletonList(new Record<>(testEvent)); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, null); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, null); pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testDocumentIdField); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List docIds = getSearchResponseDocIds(testIndexAlias); @@ -1323,9 +1335,9 @@ public void testOpenSearchRoutingField(final String testRoutingField) throws IOE final List> testRecords = Collections.singletonList(new Record<>(testEvent)); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, null); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, null); pluginSetting.getSettings().put(IndexConfiguration.ROUTING_FIELD, testRoutingField); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List routingFields = getSearchResponseRoutingFields(testIndexAlias); @@ -1350,11 +1362,11 @@ public void testOpenSearchRouting(final String testRouting) throws IOException, final List> testRecords = Collections.singletonList(new Record<>(testEvent)); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, null); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, null); if (!testRouting.isEmpty()) { pluginSetting.getSettings().put(IndexConfiguration.ROUTING, "${"+testRouting+"}"); } - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List routingFields = getSearchResponseRoutingFields(testIndexAlias); @@ -1381,10 +1393,10 @@ public void testOpenSearchRoutingWithExpressions(final String testRouting) throw final List> testRecords = Collections.singletonList(new Record<>(testEvent)); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, null); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, null); pluginSetting.getSettings().put(IndexConfiguration.ROUTING, "${/"+testRouting+"}"); when(expressionEvaluator.isValidFormatExpression(any(String.class))).thenReturn(true); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List routingFields = getSearchResponseRoutingFields(testIndexAlias); @@ -1409,10 +1421,10 @@ public void testOpenSearchRoutingWithMixedExpressions(final String testRouting) final String prefix = RandomStringUtils.randomAlphabetic(5); final String suffix = RandomStringUtils.randomAlphabetic(6); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, null); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, null); pluginSetting.getSettings().put(IndexConfiguration.ROUTING, prefix+"-${/"+testRouting+"}-"+suffix); when(expressionEvaluator.isValidFormatExpression(any(String.class))).thenReturn(true); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final String expectedRouting = prefix+"-"+routing+"-"+suffix; @@ -1441,8 +1453,8 @@ public void testOpenSearchDynamicIndex(final String testIndex) throws IOExceptio final List> testRecords = Collections.singletonList(new Record<>(testEvent)); - final PluginSetting pluginSetting = generatePluginSetting(null, dynamicTestIndexAlias, null); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, dynamicTestIndexAlias, null); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List> retSources = getSearchResponseDocSources(testIndexAlias); assertThat(retSources.size(), equalTo(1)); @@ -1474,8 +1486,8 @@ public void testOpenSearchDynamicIndexWithDate(final String testIndex, final Str final List> testRecords = Collections.singletonList(new Record<>(testEvent)); - final PluginSetting pluginSetting = generatePluginSetting(null, dynamicTestIndexAlias, null); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, dynamicTestIndexAlias, null); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List> retSources = getSearchResponseDocSources(expectedIndexAlias); assertThat(retSources.size(), equalTo(1)); @@ -1510,8 +1522,8 @@ public void testOpenSearchDynamicIndexWithDateNotAsSuffix( final List> testRecords = Collections.singletonList(new Record<>(testEvent)); - final PluginSetting pluginSetting = generatePluginSetting(null, dynamicTestIndexAlias, null); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, dynamicTestIndexAlias, null); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List> retSources = getSearchResponseDocSources(expectedIndexAlias); assertThat(retSources.size(), equalTo(1)); @@ -1538,8 +1550,8 @@ public void testOpenSearchIndexWithDate(final String testDatePattern) throws IOE final List> testRecords = Collections.singletonList(new Record<>(testEvent)); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, null); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, null); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List> retSources = getSearchResponseDocSources(expectedIndexName); assertThat(retSources.size(), equalTo(1)); @@ -1566,8 +1578,8 @@ public void testOpenSearchIndexWithDateNotAsSuffix(final String testIndexAlias) final List> testRecords = Collections.singletonList(new Record<>(testEvent)); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, null); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, null); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List> retSources = getSearchResponseDocSources(expectedIndexName); assertThat(retSources.size(), equalTo(1)); @@ -1579,16 +1591,16 @@ public void testOpenSearchIndexWithDateNotAsSuffix(final String testIndexAlias) public void testOpenSearchIndexWithInvalidDate() throws IOException, InterruptedException { String invalidDatePattern = "yyyy-MM-dd HH:ss:mm"; final String invalidTestIndexAlias = "test-index-%{" + invalidDatePattern + "}"; - final PluginSetting pluginSetting = generatePluginSetting(null, invalidTestIndexAlias, null); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, false); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, invalidTestIndexAlias, null); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, false); Assert.assertThrows(IllegalArgumentException.class, () -> sink.doInitialize()); } @Test public void testOpenSearchIndexWithInvalidChars() throws IOException, InterruptedException { final String invalidTestIndexAlias = "test#-index"; - final PluginSetting pluginSetting = generatePluginSetting(null, invalidTestIndexAlias, null); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, false); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, invalidTestIndexAlias, null); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, false); Assert.assertThrows(RuntimeException.class, () -> sink.doInitialize()); } @@ -1614,8 +1626,8 @@ public void testOutputManagementDisabled() throws IOException, InterruptedExcept metadata.put(IndexConfiguration.INDEX_TYPE, IndexType.MANAGEMENT_DISABLED.getValue()); metadata.put(AUTHENTICATION, Map.of(USERNAME, username, PASSWORD, password)); metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); - final PluginSetting pluginSetting = generatePluginSettingByMetadata(metadata); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); final String testTemplateFile = Objects.requireNonNull( getClass().getClassLoader().getResource("management-disabled-index-template.json")).getFile(); @@ -1657,33 +1669,38 @@ private Map initializeConfigurationMetadata(final String indexTy return metadata; } - private PluginSetting generatePluginSetting(final String indexType, final String indexAlias, - final String templateFilePath) { + private OpenSearchSinkConfig generateOpenSearchSinkConfig(final String indexType, final String indexAlias, + final String templateFilePath) throws JsonProcessingException { final Map metadata = initializeConfigurationMetadata(indexType, indexAlias, templateFilePath); - return generatePluginSettingByMetadata(metadata); + return generateOpenSearchSinkConfigByMetadata(metadata); } - private PluginSetting generatePluginSetting(final String indexType, final String indexAlias, + private OpenSearchSinkConfig generateOpenSearchSinkConfig(final String indexType, final String indexAlias, final String templateFilePath, final boolean estimateBulkSizeUsingCompression, - final boolean requestCompressionEnabled) { + final boolean requestCompressionEnabled) throws JsonProcessingException { final Map metadata = initializeConfigurationMetadata(indexType, indexAlias, templateFilePath); metadata.put(IndexConfiguration.ESTIMATE_BULK_SIZE_USING_COMPRESSION, estimateBulkSizeUsingCompression); metadata.put(ConnectionConfiguration.REQUEST_COMPRESSION_ENABLED, requestCompressionEnabled); - return generatePluginSettingByMetadata(metadata); + return generateOpenSearchSinkConfigByMetadata(metadata); } - private PluginSetting generatePluginSetting(final String indexType, final String indexAlias, + private OpenSearchSinkConfig generateOpenSearchSinkConfig(final String indexType, final String indexAlias, final String templateType, - final String templateFilePath) { + final String templateFilePath) throws JsonProcessingException { final Map metadata = initializeConfigurationMetadata(indexType, indexAlias, templateFilePath); metadata.put(IndexConfiguration.TEMPLATE_TYPE, templateType); - return generatePluginSettingByMetadata(metadata); + return generateOpenSearchSinkConfigByMetadata(metadata); } - private PluginSetting generatePluginSettingByMetadata(final Map configurationMetadata) { - final PluginSetting pluginSetting = new PluginSetting(PLUGIN_NAME, configurationMetadata); - pluginSetting.setPipelineName(PIPELINE_NAME); - return pluginSetting; + private OpenSearchSinkConfig generateOpenSearchSinkConfigByMetadata(final Map configurationMetadata) throws JsonProcessingException { + objectMapper = new ObjectMapper(); + List hosts = new ArrayList<>(); + hosts.add("test_host"); + configurationMetadata.put("hosts", hosts); + String json = new ObjectMapper().writeValueAsString(configurationMetadata); + OpenSearchSinkConfig openSearchSinkConfig = objectMapper.readValue(json, OpenSearchSinkConfig.class); + + return openSearchSinkConfig; } private String generateCustomRecordJson(final String idField, final String documentId) throws IOException { From 0024ef0be00e108aecf29564f71fed963e73159b Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Wed, 18 Dec 2024 13:59:17 -0800 Subject: [PATCH 16/62] build.gradle change? Signed-off-by: Maxwell Brown --- build.gradle | 2 -- 1 file changed, 2 deletions(-) diff --git a/build.gradle b/build.gradle index e8fe2d06d3..3360da81b5 100644 --- a/build.gradle +++ b/build.gradle @@ -74,8 +74,6 @@ subprojects { implementation platform('com.fasterxml.jackson:jackson-bom:2.17.2') implementation platform('org.eclipse.jetty:jetty-bom:9.4.53.v20231009') implementation platform('io.micrometer:micrometer-bom:1.10.5') - implementation platform('com.fasterxml.jackson:jackson-bom:2.17.2') - implementation platform('org.eclipse.jetty:jetty-bom:9.4.53.v20231009') implementation libs.guava.core implementation libs.slf4j.api testImplementation testLibs.bundles.junit From b1d54252906c3062b434726d4a4c9d3123e157b2 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Wed, 18 Dec 2024 16:19:50 -0800 Subject: [PATCH 17/62] remove hosts in integration test metadata Signed-off-by: Maxwell Brown --- .../plugins/sink/opensearch/OpenSearchSinkIT.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index ed5e8dfe03..b57a656454 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -141,9 +141,6 @@ public class OpenSearchSinkIT { @Mock private ExpressionEvaluator expressionEvaluator; - @Mock - private OpenSearchSinkConfig openSearchSinkConfig; - @Mock private PipelineDescription pipelineDescription; @@ -1694,9 +1691,6 @@ private OpenSearchSinkConfig generateOpenSearchSinkConfig(final String indexType private OpenSearchSinkConfig generateOpenSearchSinkConfigByMetadata(final Map configurationMetadata) throws JsonProcessingException { objectMapper = new ObjectMapper(); - List hosts = new ArrayList<>(); - hosts.add("test_host"); - configurationMetadata.put("hosts", hosts); String json = new ObjectMapper().writeValueAsString(configurationMetadata); OpenSearchSinkConfig openSearchSinkConfig = objectMapper.readValue(json, OpenSearchSinkConfig.class); From c949f910d41e7aded1c92b6b599a11386589fb0f Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Wed, 18 Dec 2024 16:36:13 -0800 Subject: [PATCH 18/62] changing enforcement for required and non required fields Signed-off-by: Maxwell Brown --- .../sink/opensearch/configuration/ActionConfiguration.java | 2 ++ .../configuration/AwsAuthenticationConfiguration.java | 5 ++--- .../sink/opensearch/configuration/ServerlessOptions.java | 4 ++++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ActionConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ActionConfiguration.java index 4f1315d6e6..f66b816d7a 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ActionConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ActionConfiguration.java @@ -2,12 +2,14 @@ import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.Size; import lombok.Getter; import org.apache.commons.lang3.EnumUtils; import org.opensearch.dataprepper.model.opensearch.OpenSearchBulkActions; public class ActionConfiguration { @Getter + @Size(min = 1, message = "type cannot be empty") @JsonProperty("type") private String type; diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AwsAuthenticationConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AwsAuthenticationConfiguration.java index b3d428b96f..1c61f14ae6 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AwsAuthenticationConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AwsAuthenticationConfiguration.java @@ -15,15 +15,14 @@ public class AwsAuthenticationConfiguration { @JsonProperty("region") - @Size(min = 1, message = "Region cannot be empty string") private String awsRegion; @JsonProperty("sts_role_arn") - @Size(min = 20, max = 2048, message = "awsStsRoleArn length should be between 1 and 2048 characters") + @Size(max = 2048, message = "awsStsRoleArn length should be less than 2048 characters") private String awsStsRoleArn; @JsonProperty("sts_external_id") - @Size(min = 2, max = 1224, message = "awsStsExternalId length should be between 2 and 1224 characters") + @Size(max = 1224, message = "awsStsExternalId length should be between less than 1224 characters") private String awsStsExternalId; @JsonProperty("sts_header_overrides") diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ServerlessOptions.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ServerlessOptions.java index ddd3d0aa67..a274f47519 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ServerlessOptions.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ServerlessOptions.java @@ -1,15 +1,19 @@ package org.opensearch.dataprepper.plugins.sink.opensearch.configuration; import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.Size; public class ServerlessOptions { + @Size(min = 1, message = "network_policy_name cannot be empty") @JsonProperty("network_policy_name") private String networkPolicyName; + @Size(min = 1, message = "collection_name cannot be empty") @JsonProperty("collection_name") private String collectionName; + @Size(min = 1, message = "vpce_id cannot be empty") @JsonProperty("vpce_id") private String vpceId; From 53398fb104d2b829ac6c9931fb3b80e77e5a9c68 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Wed, 18 Dec 2024 16:46:10 -0800 Subject: [PATCH 19/62] reset awsauthconfig to normal Signed-off-by: Maxwell Brown --- .../configuration/AwsAuthenticationConfiguration.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AwsAuthenticationConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AwsAuthenticationConfiguration.java index 1c61f14ae6..b3d428b96f 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AwsAuthenticationConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AwsAuthenticationConfiguration.java @@ -15,14 +15,15 @@ public class AwsAuthenticationConfiguration { @JsonProperty("region") + @Size(min = 1, message = "Region cannot be empty string") private String awsRegion; @JsonProperty("sts_role_arn") - @Size(max = 2048, message = "awsStsRoleArn length should be less than 2048 characters") + @Size(min = 20, max = 2048, message = "awsStsRoleArn length should be between 1 and 2048 characters") private String awsStsRoleArn; @JsonProperty("sts_external_id") - @Size(max = 1224, message = "awsStsExternalId length should be between less than 1224 characters") + @Size(min = 2, max = 1224, message = "awsStsExternalId length should be between 2 and 1224 characters") private String awsStsExternalId; @JsonProperty("sts_header_overrides") From be265c01b07c2b060adcec8160fd4ff571856831 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Wed, 18 Dec 2024 16:51:19 -0800 Subject: [PATCH 20/62] replace authentication field from unit tests Signed-off-by: Maxwell Brown --- .../dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index b57a656454..3bcdccd0ba 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -1657,7 +1657,8 @@ private Map initializeConfigurationMetadata(final String indexTy final String user = System.getProperty("tests.opensearch.user"); final String password = System.getProperty("tests.opensearch.password"); if (user != null) { - metadata.put(AUTHENTICATION, Map.of(USERNAME, user, PASSWORD, password)); + metadata.put(USERNAME, user); + metadata.put(PASSWORD, password); } final String distributionVersion = DeclaredOpenSearchVersion.OPENDISTRO_0_10.compareTo( OpenSearchIntegrationHelper.getVersion()) >= 0 ? From 5f9b9f6ded47c696f31a327b0ba104b56e90079a Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Wed, 18 Dec 2024 17:07:25 -0800 Subject: [PATCH 21/62] document_id_field to document_id since deprecated Signed-off-by: Maxwell Brown --- .../sink/opensearch/OpenSearchSinkIT.java | 26 +++++++++---------- .../opensearch/index/IndexConfiguration.java | 1 - 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index 3bcdccd0ba..14597bde95 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -824,7 +824,7 @@ public void testOutputCustomIndex() throws IOException, InterruptedException { final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List> retSources = getSearchResponseDocSources(testIndexAlias); @@ -850,7 +850,7 @@ public void testOpenSearchBulkActionsCreate() throws IOException, InterruptedExc final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); pluginSetting.getSettings().put(IndexConfiguration.ACTION, OpenSearchBulkActions.CREATE.toString()); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); @@ -877,7 +877,7 @@ public void testOpenSearchBulkActionsCreateWithExpression() throws IOException, final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); Event event = (Event) testRecords.get(0).getData(); event.getMetadata().setAttribute("action", "create"); final String actionFormatExpression = "${getMetadata(\"action\")}"; @@ -910,7 +910,7 @@ public void testOpenSearchBulkActionsCreateWithInvalidExpression() throws IOExce final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); Event event = (Event) testRecords.get(0).getData(); event.getMetadata().setAttribute("action", "unknown"); final String actionFormatExpression = "${getMetadata(\"action\")}"; @@ -936,7 +936,7 @@ public void testBulkActionCreateWithActions() throws IOException, InterruptedExc final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.CREATE.toString()); @@ -969,7 +969,7 @@ public void testBulkActionUpdateWithActions() throws IOException, InterruptedExc List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson2(testIdField, testId, "name", "value1"))); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.CREATE.toString()); @@ -1027,7 +1027,7 @@ public void testBulkActionUpdateWithDocumentRootKey() throws IOException, Interr final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ROOT_KEY, documentRootKey); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); List> aList = new ArrayList<>(); Map actionMap = new HashMap<>(); actionMap.put("type", OpenSearchBulkActions.CREATE.toString()); @@ -1088,7 +1088,7 @@ public void testBulkActionUpsertWithActionsAndNoCreate() throws IOException, Int aList.add(actionMap); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); pluginSetting.getSettings().put(IndexConfiguration.ACTIONS, aList); OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); @@ -1114,7 +1114,7 @@ public void testBulkActionUpsertWithActions() throws IOException, InterruptedExc List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson2(testIdField, testId, "name", "value1"))); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.CREATE.toString()); @@ -1162,7 +1162,7 @@ public void testBulkActionUpsertWithoutCreate() throws IOException, InterruptedE final String testId = "foo"; List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson3(testIdField, testId, "name", "value1", "newKey", "newValue"))); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.UPSERT.toString()); @@ -1198,7 +1198,7 @@ public void testBulkActionDeleteWithActions() throws IOException, InterruptedExc List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.DELETE.toString()); @@ -1308,7 +1308,7 @@ public void testOpenSearchDocumentId(final String testDocumentIdField) throws IO final List> testRecords = Collections.singletonList(new Record<>(testEvent)); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, null); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testDocumentIdField); + pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testDocumentIdField); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); @@ -1622,7 +1622,7 @@ public void testOutputManagementDisabled() throws IOException, InterruptedExcept final Map metadata = initializeConfigurationMetadata(null, testIndexAlias, null); metadata.put(IndexConfiguration.INDEX_TYPE, IndexType.MANAGEMENT_DISABLED.getValue()); metadata.put(AUTHENTICATION, Map.of(USERNAME, username, PASSWORD, password)); - metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java index 172f17c4b4..ebd1240655 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java @@ -54,7 +54,6 @@ public class IndexConfiguration { public static final String ESTIMATE_BULK_SIZE_USING_COMPRESSION = "estimate_bulk_size_using_compression"; public static final String MAX_LOCAL_COMPRESSIONS_FOR_ESTIMATION = "max_local_compressions_for_estimation"; public static final String FLUSH_TIMEOUT = "flush_timeout"; - public static final String DOCUMENT_ID_FIELD = "document_id_field"; public static final String DOCUMENT_ID = "document_id"; public static final String ROUTING_FIELD = "routing_field"; public static final String ROUTING = "routing"; From 8450dfe9f07c929decf7ce809d2e41e4b6f822c8 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Wed, 18 Dec 2024 21:55:59 -0800 Subject: [PATCH 22/62] replace pluginSetting dependent parts of integration tests Signed-off-by: Maxwell Brown --- .../sink/opensearch/OpenSearchSinkIT.java | 126 +++++++++++------- 1 file changed, 78 insertions(+), 48 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index 14597bde95..095c649378 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -422,13 +422,14 @@ public void testOutputRawSpanWithDLQ(final boolean estimateBulkSizeUsingCompress @SuppressWarnings("unchecked") final Map expData = mapper.readValue(testDoc2, Map.class); final List> testRecords = Arrays.asList(jsonStringToRecord(testDoc1), jsonStringToRecord(testDoc2)); - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(IndexType.TRACE_ANALYTICS_RAW.getValue(), null, null, + Map metadata = initializeConfigurationMetadata(IndexType.TRACE_ANALYTICS_RAW.getValue(), null, null, estimateBulkSizeUsingCompression, isRequestCompressionEnabled); // generate temporary directory for dlq file final File tempDirectory = Files.createTempDirectory("").toFile(); // add dlq file path into setting final String expDLQFile = tempDirectory.getAbsolutePath() + "/test-dlq.txt"; - pluginSetting.getSettings().put(RetryConfiguration.DLQ_FILE, expDLQFile); + metadata.put(RetryConfiguration.DLQ_FILE, expDLQFile); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); @@ -823,8 +824,9 @@ public void testOutputCustomIndex() throws IOException, InterruptedException { final String testIdField = "someId"; final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); + metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List> retSources = getSearchResponseDocSources(testIndexAlias); @@ -849,9 +851,10 @@ public void testOpenSearchBulkActionsCreate() throws IOException, InterruptedExc final String testIdField = "someId"; final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); - pluginSetting.getSettings().put(IndexConfiguration.ACTION, OpenSearchBulkActions.CREATE.toString()); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); + metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField); + metadata.put(IndexConfiguration.ACTION, OpenSearchBulkActions.CREATE.toString()); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List> retSources = getSearchResponseDocSources(testIndexAlias); @@ -876,15 +879,16 @@ public void testOpenSearchBulkActionsCreateWithExpression() throws IOException, final String testIdField = "someId"; final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); + metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField); Event event = (Event) testRecords.get(0).getData(); event.getMetadata().setAttribute("action", "create"); final String actionFormatExpression = "${getMetadata(\"action\")}"; when(expressionEvaluator.isValidFormatExpression(actionFormatExpression)).thenReturn(true); when(expressionEvaluator.isValidExpressionStatement("getMetadata(\"action\")")).thenReturn(true); when(expressionEvaluator.evaluate("getMetadata(\"action\")", event)).thenReturn(event.getMetadata().getAttribute("action")); - pluginSetting.getSettings().put(IndexConfiguration.ACTION, actionFormatExpression); + metadata.put(IndexConfiguration.ACTION, actionFormatExpression); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List> retSources = getSearchResponseDocSources(testIndexAlias); @@ -909,15 +913,16 @@ public void testOpenSearchBulkActionsCreateWithInvalidExpression() throws IOExce final String testIdField = "someId"; final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); + metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField); Event event = (Event) testRecords.get(0).getData(); event.getMetadata().setAttribute("action", "unknown"); final String actionFormatExpression = "${getMetadata(\"action\")}"; when(expressionEvaluator.isValidFormatExpression(actionFormatExpression)).thenReturn(true); when(expressionEvaluator.isValidExpressionStatement("getMetadata(\"action\")")).thenReturn(true); when(expressionEvaluator.evaluate("getMetadata(\"action\")", event)).thenReturn(event.getMetadata().getAttribute("action")); - pluginSetting.getSettings().put(IndexConfiguration.ACTION, actionFormatExpression); + metadata.put(IndexConfiguration.ACTION, actionFormatExpression); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List> retSources = getSearchResponseDocSources(testIndexAlias); @@ -935,13 +940,14 @@ public void testBulkActionCreateWithActions() throws IOException, InterruptedExc final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); + metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.CREATE.toString()); aList.add(aMap); - pluginSetting.getSettings().put(IndexConfiguration.ACTIONS, aList); + metadata.put(IndexConfiguration.ACTIONS, aList); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List> retSources = getSearchResponseDocSources(testIndexAlias); @@ -968,13 +974,14 @@ public void testBulkActionUpdateWithActions() throws IOException, InterruptedExc final String testId = "foo"; List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson2(testIdField, testId, "name", "value1"))); - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); + metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.CREATE.toString()); aList.add(aMap); - pluginSetting.getSettings().put(IndexConfiguration.ACTIONS, aList); + metadata.put(IndexConfiguration.ACTIONS, aList); + OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); List> retSources = getSearchResponseDocSources(testIndexAlias); @@ -994,7 +1001,8 @@ public void testBulkActionUpdateWithActions() throws IOException, InterruptedExc aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.UPDATE.toString()); aList.add(aMap); - pluginSetting.getSettings().put(IndexConfiguration.ACTIONS, aList); + metadata.put(IndexConfiguration.ACTIONS, aList); + openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); retSources = getSearchResponseDocSources(testIndexAlias); @@ -1024,17 +1032,18 @@ public void testBulkActionUpdateWithDocumentRootKey() throws IOException, Interr List> testRecords = Collections.singletonList(jsonStringToRecord(createJsonEvent)); - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ROOT_KEY, documentRootKey); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); + metadata.put(IndexConfiguration.DOCUMENT_ROOT_KEY, documentRootKey); + metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField); List> aList = new ArrayList<>(); Map actionMap = new HashMap<>(); actionMap.put("type", OpenSearchBulkActions.CREATE.toString()); aList.add(actionMap); - pluginSetting.getSettings().put(IndexConfiguration.ACTIONS, aList); + metadata.put(IndexConfiguration.ACTIONS, aList); + OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); List> retSources = getSearchResponseDocSources(testIndexAlias); @@ -1059,7 +1068,8 @@ public void testBulkActionUpdateWithDocumentRootKey() throws IOException, Interr actionMap = new HashMap<>(); actionMap.put("type", OpenSearchBulkActions.UPDATE.toString()); aList.add(actionMap); - pluginSetting.getSettings().put(IndexConfiguration.ACTIONS, aList); + metadata.put(IndexConfiguration.ACTIONS, aList); + openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); retSources = getSearchResponseDocSources(testIndexAlias); @@ -1087,9 +1097,10 @@ public void testBulkActionUpsertWithActionsAndNoCreate() throws IOException, Int actionMap.put("type", OpenSearchBulkActions.UPSERT.toString()); aList.add(actionMap); - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); - pluginSetting.getSettings().put(IndexConfiguration.ACTIONS, aList); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); + metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField); + metadata.put(IndexConfiguration.ACTIONS, aList); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); @@ -1113,13 +1124,15 @@ public void testBulkActionUpsertWithActions() throws IOException, InterruptedExc final String testId = "foo"; List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson2(testIdField, testId, "name", "value1"))); - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); + metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.CREATE.toString()); aList.add(aMap); - pluginSetting.getSettings().put(IndexConfiguration.ACTIONS, aList); + metadata.put(IndexConfiguration.ACTIONS, aList); + OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); List> retSources = getSearchResponseDocSources(testIndexAlias); @@ -1139,7 +1152,8 @@ public void testBulkActionUpsertWithActions() throws IOException, InterruptedExc aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.UPSERT.toString()); aList.add(aMap); - pluginSetting.getSettings().put(IndexConfiguration.ACTIONS, aList); + metadata.put(IndexConfiguration.ACTIONS, aList); + openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); retSources = getSearchResponseDocSources(testIndexAlias); @@ -1161,13 +1175,14 @@ public void testBulkActionUpsertWithoutCreate() throws IOException, InterruptedE final String testIdField = "someId"; final String testId = "foo"; List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson3(testIdField, testId, "name", "value1", "newKey", "newValue"))); - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); + metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.UPSERT.toString()); aList.add(aMap); - pluginSetting.getSettings().put(IndexConfiguration.ACTIONS, aList); + metadata.put(IndexConfiguration.ACTIONS, aList); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); List> retSources = getSearchResponseDocSources(testIndexAlias); @@ -1197,13 +1212,14 @@ public void testBulkActionDeleteWithActions() throws IOException, InterruptedExc final String testId = "foo"; List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); + metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.DELETE.toString()); aList.add(aMap); - pluginSetting.getSettings().put(IndexConfiguration.ACTIONS, aList); + metadata.put(IndexConfiguration.ACTIONS, aList); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); List> retSources = getSearchResponseDocSources(testIndexAlias); @@ -1307,8 +1323,9 @@ public void testOpenSearchDocumentId(final String testDocumentIdField) throws IO final List> testRecords = Collections.singletonList(new Record<>(testEvent)); - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, null); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testDocumentIdField); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, null); + metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentIdField); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); @@ -1332,8 +1349,9 @@ public void testOpenSearchRoutingField(final String testRoutingField) throws IOE final List> testRecords = Collections.singletonList(new Record<>(testEvent)); - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, null); - pluginSetting.getSettings().put(IndexConfiguration.ROUTING_FIELD, testRoutingField); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, null); + metadata.put(IndexConfiguration.ROUTING_FIELD, testRoutingField); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); @@ -1359,10 +1377,11 @@ public void testOpenSearchRouting(final String testRouting) throws IOException, final List> testRecords = Collections.singletonList(new Record<>(testEvent)); - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, null); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, null); if (!testRouting.isEmpty()) { - pluginSetting.getSettings().put(IndexConfiguration.ROUTING, "${"+testRouting+"}"); + metadata.put(IndexConfiguration.ROUTING, "${"+testRouting+"}"); } + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); @@ -1390,9 +1409,10 @@ public void testOpenSearchRoutingWithExpressions(final String testRouting) throw final List> testRecords = Collections.singletonList(new Record<>(testEvent)); - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, null); - pluginSetting.getSettings().put(IndexConfiguration.ROUTING, "${/"+testRouting+"}"); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, null); + metadata.put(IndexConfiguration.ROUTING, "${/"+testRouting+"}"); when(expressionEvaluator.isValidFormatExpression(any(String.class))).thenReturn(true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); @@ -1418,9 +1438,10 @@ public void testOpenSearchRoutingWithMixedExpressions(final String testRouting) final String prefix = RandomStringUtils.randomAlphabetic(5); final String suffix = RandomStringUtils.randomAlphabetic(6); - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, null); - pluginSetting.getSettings().put(IndexConfiguration.ROUTING, prefix+"-${/"+testRouting+"}-"+suffix); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, null); + metadata.put(IndexConfiguration.ROUTING, prefix+"-${/"+testRouting+"}-"+suffix); when(expressionEvaluator.isValidFormatExpression(any(String.class))).thenReturn(true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final String expectedRouting = prefix+"-"+routing+"-"+suffix; @@ -1667,6 +1688,15 @@ private Map initializeConfigurationMetadata(final String indexTy return metadata; } + private Map initializeConfigurationMetadata(final String indexType, final String indexAlias, + final String templateFilePath, final boolean estimateBulkSizeUsingCompression, + final boolean requestCompressionEnabled) throws JsonProcessingException { + final Map metadata = initializeConfigurationMetadata(indexType, indexAlias, templateFilePath); + metadata.put(IndexConfiguration.ESTIMATE_BULK_SIZE_USING_COMPRESSION, estimateBulkSizeUsingCompression); + metadata.put(ConnectionConfiguration.REQUEST_COMPRESSION_ENABLED, requestCompressionEnabled); + return metadata; + } + private OpenSearchSinkConfig generateOpenSearchSinkConfig(final String indexType, final String indexAlias, final String templateFilePath) throws JsonProcessingException { final Map metadata = initializeConfigurationMetadata(indexType, indexAlias, templateFilePath); From fcbbaf66c1da442b31d57a974bf97a7be28f31d8 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Thu, 19 Dec 2024 11:01:23 -0800 Subject: [PATCH 23/62] pipeline name missing from integration tests Signed-off-by: Maxwell Brown --- .../dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index 095c649378..a6d52eaa74 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -151,6 +151,7 @@ public class OpenSearchSinkIT { private PluginConfigObservable pluginConfigObservable; public OpenSearchSink createObjectUnderTest(OpenSearchSinkConfig openSearchSinkConfig, boolean doInitialize) { + when(pipelineDescription.getPipelineName()).thenReturn(PIPELINE_NAME); OpenSearchSink sink = new OpenSearchSink( pluginSetting, null, expressionEvaluator, awsCredentialsSupplier, pipelineDescription, pluginConfigObservable, openSearchSinkConfig); if (doInitialize) { @@ -163,6 +164,7 @@ public OpenSearchSink createObjectUnderTestWithSinkContext(OpenSearchSinkConfig sinkContext = mock(SinkContext.class); testTagsTargetKey = RandomStringUtils.randomAlphabetic(5); when(sinkContext.getTagsTargetKey()).thenReturn(testTagsTargetKey); + when(pipelineDescription.getPipelineName()).thenReturn(PIPELINE_NAME); OpenSearchSink sink = new OpenSearchSink( pluginSetting, null, expressionEvaluator, awsCredentialsSupplier, pipelineDescription, pluginConfigObservable, openSearchSinkConfig); if (doInitialize) { From 11b803be47af711b3482a4356beb52e3d9ed1210 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Thu, 19 Dec 2024 11:36:11 -0800 Subject: [PATCH 24/62] find what is null Signed-off-by: Maxwell Brown --- .../plugins/sink/opensearch/OpenSearchSinkIT.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index a6d52eaa74..79e62dd447 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -90,6 +90,7 @@ import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.closeTo; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.params.provider.Arguments.arguments; import static org.mockito.ArgumentMatchers.any; @@ -132,9 +133,6 @@ public class OpenSearchSinkIT { ObjectMapper objectMapper; - @Mock - private PluginFactory pluginFactory; - @Mock private AwsCredentialsSupplier awsCredentialsSupplier; @@ -152,6 +150,7 @@ public class OpenSearchSinkIT { public OpenSearchSink createObjectUnderTest(OpenSearchSinkConfig openSearchSinkConfig, boolean doInitialize) { when(pipelineDescription.getPipelineName()).thenReturn(PIPELINE_NAME); + assertNotNull(openSearchSinkConfig); OpenSearchSink sink = new OpenSearchSink( pluginSetting, null, expressionEvaluator, awsCredentialsSupplier, pipelineDescription, pluginConfigObservable, openSearchSinkConfig); if (doInitialize) { @@ -166,7 +165,7 @@ public OpenSearchSink createObjectUnderTestWithSinkContext(OpenSearchSinkConfig when(sinkContext.getTagsTargetKey()).thenReturn(testTagsTargetKey); when(pipelineDescription.getPipelineName()).thenReturn(PIPELINE_NAME); OpenSearchSink sink = new OpenSearchSink( - pluginSetting, null, expressionEvaluator, awsCredentialsSupplier, pipelineDescription, pluginConfigObservable, openSearchSinkConfig); + pluginSetting, sinkContext, expressionEvaluator, awsCredentialsSupplier, pipelineDescription, pluginConfigObservable, openSearchSinkConfig); if (doInitialize) { sink.doInitialize(); } From 9378889095eed094be9da5299d7ff83b7f80fb1b Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Thu, 19 Dec 2024 11:44:16 -0800 Subject: [PATCH 25/62] initialize pipelineDescription Mock Signed-off-by: Maxwell Brown --- .../dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index 79e62dd447..2fdafb03b1 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -150,7 +150,6 @@ public class OpenSearchSinkIT { public OpenSearchSink createObjectUnderTest(OpenSearchSinkConfig openSearchSinkConfig, boolean doInitialize) { when(pipelineDescription.getPipelineName()).thenReturn(PIPELINE_NAME); - assertNotNull(openSearchSinkConfig); OpenSearchSink sink = new OpenSearchSink( pluginSetting, null, expressionEvaluator, awsCredentialsSupplier, pipelineDescription, pluginConfigObservable, openSearchSinkConfig); if (doInitialize) { @@ -176,6 +175,7 @@ public OpenSearchSink createObjectUnderTestWithSinkContext(OpenSearchSinkConfig public void setup() { pluginConfigObservable = mock(PluginConfigObservable.class); expressionEvaluator = mock(ExpressionEvaluator.class); + pipelineDescription = mock(PipelineDescription.class); when(expressionEvaluator.isValidExpressionStatement(any(String.class))).thenReturn(false); } From 96ea7e13c2024304161689080235f9d144e4fa08 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Thu, 19 Dec 2024 11:55:30 -0800 Subject: [PATCH 26/62] initialize pluginSetting Mock Signed-off-by: Maxwell Brown --- .../dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index 2fdafb03b1..a6f6612d27 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -150,6 +150,7 @@ public class OpenSearchSinkIT { public OpenSearchSink createObjectUnderTest(OpenSearchSinkConfig openSearchSinkConfig, boolean doInitialize) { when(pipelineDescription.getPipelineName()).thenReturn(PIPELINE_NAME); + assertNotNull(openSearchSinkConfig); OpenSearchSink sink = new OpenSearchSink( pluginSetting, null, expressionEvaluator, awsCredentialsSupplier, pipelineDescription, pluginConfigObservable, openSearchSinkConfig); if (doInitialize) { @@ -176,6 +177,7 @@ public void setup() { pluginConfigObservable = mock(PluginConfigObservable.class); expressionEvaluator = mock(ExpressionEvaluator.class); pipelineDescription = mock(PipelineDescription.class); + pluginSetting = mock(PluginSetting.class); when(expressionEvaluator.isValidExpressionStatement(any(String.class))).thenReturn(false); } From d6aa92a3a7442d0254834dd3fc4aafbb9696663b Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Thu, 19 Dec 2024 12:05:47 -0800 Subject: [PATCH 27/62] add pluginSetting name mock Signed-off-by: Maxwell Brown --- .../dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index a6f6612d27..ce1e443f1a 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -150,7 +150,7 @@ public class OpenSearchSinkIT { public OpenSearchSink createObjectUnderTest(OpenSearchSinkConfig openSearchSinkConfig, boolean doInitialize) { when(pipelineDescription.getPipelineName()).thenReturn(PIPELINE_NAME); - assertNotNull(openSearchSinkConfig); + when(pluginSetting.getName()).thenReturn(PLUGIN_NAME); OpenSearchSink sink = new OpenSearchSink( pluginSetting, null, expressionEvaluator, awsCredentialsSupplier, pipelineDescription, pluginConfigObservable, openSearchSinkConfig); if (doInitialize) { @@ -164,6 +164,7 @@ public OpenSearchSink createObjectUnderTestWithSinkContext(OpenSearchSinkConfig testTagsTargetKey = RandomStringUtils.randomAlphabetic(5); when(sinkContext.getTagsTargetKey()).thenReturn(testTagsTargetKey); when(pipelineDescription.getPipelineName()).thenReturn(PIPELINE_NAME); + when(pluginSetting.getName()).thenReturn(PLUGIN_NAME); OpenSearchSink sink = new OpenSearchSink( pluginSetting, sinkContext, expressionEvaluator, awsCredentialsSupplier, pipelineDescription, pluginConfigObservable, openSearchSinkConfig); if (doInitialize) { From c53fb659dcf2d9baf113cdd477dbeba5743725b2 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Thu, 19 Dec 2024 12:17:15 -0800 Subject: [PATCH 28/62] me when I see stack trace Signed-off-by: Maxwell Brown --- .../dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index ce1e443f1a..51cb01074b 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -150,6 +150,7 @@ public class OpenSearchSinkIT { public OpenSearchSink createObjectUnderTest(OpenSearchSinkConfig openSearchSinkConfig, boolean doInitialize) { when(pipelineDescription.getPipelineName()).thenReturn(PIPELINE_NAME); + when(pluginSetting.getPipelineName()).thenReturn(PIPELINE_NAME); when(pluginSetting.getName()).thenReturn(PLUGIN_NAME); OpenSearchSink sink = new OpenSearchSink( pluginSetting, null, expressionEvaluator, awsCredentialsSupplier, pipelineDescription, pluginConfigObservable, openSearchSinkConfig); @@ -164,6 +165,7 @@ public OpenSearchSink createObjectUnderTestWithSinkContext(OpenSearchSinkConfig testTagsTargetKey = RandomStringUtils.randomAlphabetic(5); when(sinkContext.getTagsTargetKey()).thenReturn(testTagsTargetKey); when(pipelineDescription.getPipelineName()).thenReturn(PIPELINE_NAME); + when(pluginSetting.getPipelineName()).thenReturn(PIPELINE_NAME); when(pluginSetting.getName()).thenReturn(PLUGIN_NAME); OpenSearchSink sink = new OpenSearchSink( pluginSetting, sinkContext, expressionEvaluator, awsCredentialsSupplier, pipelineDescription, pluginConfigObservable, openSearchSinkConfig); From a0af91e1602c27f5b3608abdab4dae86de6f07fe Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Thu, 19 Dec 2024 15:08:28 -0800 Subject: [PATCH 29/62] revert bulk retry strat Signed-off-by: Maxwell Brown --- .../plugins/sink/opensearch/OpenSearchSinkIT.java | 3 ++- .../plugins/sink/opensearch/BulkRetryStrategy.java | 10 +++++----- .../plugins/sink/opensearch/OpenSearchSink.java | 5 +++-- .../sink/opensearch/BulkRetryStrategyTests.java | 6 ++---- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index 51cb01074b..7f27700c73 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -1648,7 +1648,8 @@ public void testOutputManagementDisabled() throws IOException, InterruptedExcept final Map metadata = initializeConfigurationMetadata(null, testIndexAlias, null); metadata.put(IndexConfiguration.INDEX_TYPE, IndexType.MANAGEMENT_DISABLED.getValue()); - metadata.put(AUTHENTICATION, Map.of(USERNAME, username, PASSWORD, password)); + metadata.put(USERNAME, username); + metadata.put(PASSWORD, password); metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategy.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategy.java index 9deea290e6..aca748b3ef 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategy.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategy.java @@ -15,6 +15,7 @@ import org.opensearch.client.opensearch.core.BulkResponse; import org.opensearch.client.opensearch.core.bulk.BulkResponseItem; import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.AccumulatingBulkRequest; import org.opensearch.dataprepper.plugins.sink.opensearch.dlq.FailedBulkOperation; import org.opensearch.rest.RestStatus; @@ -149,16 +150,15 @@ public BulkRetryStrategy(final RequestFunction bulkRequestSupplier, - final String pipelineName, - final String pluginName) { + final PluginSetting pluginSetting) { this.requestFunction = requestFunction; this.logFailure = logFailure; this.pluginMetrics = pluginMetrics; this.bulkRequestSupplier = bulkRequestSupplier; this.maxRetries = maxRetries; - this.pipelineName = pipelineName; - this.pluginId = pluginName; - this.pluginName = pluginName; + this.pipelineName = pluginSetting.getPipelineName(); + this.pluginId = pluginSetting.getName(); + this.pluginName = pluginSetting.getName(); this.objectMapper = new ObjectMapper(); sentDocumentsCounter = pluginMetrics.counter(DOCUMENTS_SUCCESS); diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java index 09806d57af..39218bc1e4 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java @@ -129,6 +129,7 @@ public class OpenSearchSink extends AbstractSink> { private final Counter bulkRequestErrorsCounter; private final Counter invalidActionErrorsCounter; private final Counter dynamicIndexDroppedEvents; + private PluginSetting pluginSetting; private final DistributionSummary bulkRequestSizeBytesSummary; private final Counter dynamicDocumentVersionDroppedEvents; private OpenSearchClient openSearchClient; @@ -157,6 +158,7 @@ public OpenSearchSink(final PluginSetting pluginSetting, this.awsCredentialsSupplier = awsCredentialsSupplier; this.sinkContext = sinkContext != null ? sinkContext : new SinkContext(null, Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); this.expressionEvaluator = expressionEvaluator; + this.pluginSetting = pluginSetting; this.pipeline = pipelineDescription.getPipelineName(); bulkRequestTimer = pluginMetrics.timer(BULKREQUEST_LATENCY); bulkRequestErrorsCounter = pluginMetrics.counter(BULKREQUEST_ERRORS); @@ -276,8 +278,7 @@ private void doInitializeInternal() throws IOException { pluginMetrics, maxRetries, bulkRequestSupplier, - pipeline, - PLUGIN_NAME); + pluginSetting); this.initialized = true; LOG.info("Initialized OpenSearch sink"); diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategyTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategyTests.java index d30238acb6..cc05514502 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategyTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategyTests.java @@ -124,8 +124,7 @@ public BulkRetryStrategy createObjectUnderTest( pluginMetrics, Integer.MAX_VALUE, bulkRequestSupplier, - PIPELINE_NAME, - PLUGIN_NAME); + pluginSetting); } public BulkRetryStrategy createObjectUnderTest( @@ -140,8 +139,7 @@ public BulkRetryStrategy createObjectUnderTest( pluginMetrics, maxRetries, bulkRequestSupplier, - PIPELINE_NAME, - PLUGIN_NAME); + pluginSetting); } @Test From 1c9cd9ed4512e45f9d07f1b0c3010710269d01c7 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Thu, 19 Dec 2024 15:12:05 -0800 Subject: [PATCH 30/62] revert reversion Signed-off-by: Maxwell Brown --- .../plugins/sink/opensearch/BulkRetryStrategy.java | 10 +++++----- .../plugins/sink/opensearch/OpenSearchSink.java | 5 ++--- .../sink/opensearch/BulkRetryStrategyTests.java | 6 ++++-- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategy.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategy.java index aca748b3ef..9deea290e6 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategy.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategy.java @@ -15,7 +15,6 @@ import org.opensearch.client.opensearch.core.BulkResponse; import org.opensearch.client.opensearch.core.bulk.BulkResponseItem; import org.opensearch.dataprepper.metrics.PluginMetrics; -import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.AccumulatingBulkRequest; import org.opensearch.dataprepper.plugins.sink.opensearch.dlq.FailedBulkOperation; import org.opensearch.rest.RestStatus; @@ -150,15 +149,16 @@ public BulkRetryStrategy(final RequestFunction bulkRequestSupplier, - final PluginSetting pluginSetting) { + final String pipelineName, + final String pluginName) { this.requestFunction = requestFunction; this.logFailure = logFailure; this.pluginMetrics = pluginMetrics; this.bulkRequestSupplier = bulkRequestSupplier; this.maxRetries = maxRetries; - this.pipelineName = pluginSetting.getPipelineName(); - this.pluginId = pluginSetting.getName(); - this.pluginName = pluginSetting.getName(); + this.pipelineName = pipelineName; + this.pluginId = pluginName; + this.pluginName = pluginName; this.objectMapper = new ObjectMapper(); sentDocumentsCounter = pluginMetrics.counter(DOCUMENTS_SUCCESS); diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java index 39218bc1e4..09806d57af 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java @@ -129,7 +129,6 @@ public class OpenSearchSink extends AbstractSink> { private final Counter bulkRequestErrorsCounter; private final Counter invalidActionErrorsCounter; private final Counter dynamicIndexDroppedEvents; - private PluginSetting pluginSetting; private final DistributionSummary bulkRequestSizeBytesSummary; private final Counter dynamicDocumentVersionDroppedEvents; private OpenSearchClient openSearchClient; @@ -158,7 +157,6 @@ public OpenSearchSink(final PluginSetting pluginSetting, this.awsCredentialsSupplier = awsCredentialsSupplier; this.sinkContext = sinkContext != null ? sinkContext : new SinkContext(null, Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); this.expressionEvaluator = expressionEvaluator; - this.pluginSetting = pluginSetting; this.pipeline = pipelineDescription.getPipelineName(); bulkRequestTimer = pluginMetrics.timer(BULKREQUEST_LATENCY); bulkRequestErrorsCounter = pluginMetrics.counter(BULKREQUEST_ERRORS); @@ -278,7 +276,8 @@ private void doInitializeInternal() throws IOException { pluginMetrics, maxRetries, bulkRequestSupplier, - pluginSetting); + pipeline, + PLUGIN_NAME); this.initialized = true; LOG.info("Initialized OpenSearch sink"); diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategyTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategyTests.java index cc05514502..d30238acb6 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategyTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategyTests.java @@ -124,7 +124,8 @@ public BulkRetryStrategy createObjectUnderTest( pluginMetrics, Integer.MAX_VALUE, bulkRequestSupplier, - pluginSetting); + PIPELINE_NAME, + PLUGIN_NAME); } public BulkRetryStrategy createObjectUnderTest( @@ -139,7 +140,8 @@ public BulkRetryStrategy createObjectUnderTest( pluginMetrics, maxRetries, bulkRequestSupplier, - pluginSetting); + PIPELINE_NAME, + PLUGIN_NAME); } @Test From c8d6e5e9b1367fd807eec1cdec302d60845e16f7 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Fri, 20 Dec 2024 17:26:28 -0800 Subject: [PATCH 31/62] pipelineValue has to be null for documents to write. Originally pipeline is always null because indexconfiguration pipelien value was always null Signed-off-by: Maxwell Brown --- .../plugins/sink/opensearch/OpenSearchSink.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java index 09806d57af..1030b52db8 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java @@ -508,16 +508,6 @@ SerializedJson getDocument(final Event event) { } String pipelineValue = null; - if (pipeline != null) { - try { - pipelineValue = event.formatString(pipeline, expressionEvaluator); - } catch (final ExpressionEvaluationException | EventKeyNotFoundException e) { - LOG.error("Unable to construct pipeline with format {}", pipeline, e); - } - if (StringUtils.isEmpty(pipelineValue) || StringUtils.isBlank(pipelineValue)) { - pipelineValue = null; - } - } final String document = DocumentBuilder.build(event, documentRootKey, sinkContext.getTagsTargetKey(), sinkContext.getIncludeKeys(), sinkContext.getExcludeKeys()); From 21535f65235ec0369155a7434c8bcc96e2a6ca21 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Fri, 20 Dec 2024 23:36:09 -0800 Subject: [PATCH 32/62] unit test update Signed-off-by: Maxwell Brown --- .../sink/opensearch/OpenSearchSinkIT.java | 2 -- .../sink/opensearch/OpenSearchSink.java | 4 +--- .../sink/opensearch/OpenSearchSinkTest.java | 20 ------------------- 3 files changed, 1 insertion(+), 25 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index 7f27700c73..49879c0c4d 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -45,7 +45,6 @@ import org.opensearch.dataprepper.model.event.JacksonEvent; import org.opensearch.dataprepper.model.opensearch.OpenSearchBulkActions; import org.opensearch.dataprepper.model.plugin.PluginConfigObservable; -import org.opensearch.dataprepper.model.plugin.PluginFactory; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.sink.SinkContext; import org.opensearch.dataprepper.plugins.sink.opensearch.index.AbstractIndexManager; @@ -90,7 +89,6 @@ import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.closeTo; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.params.provider.Arguments.arguments; import static org.mockito.ArgumentMatchers.any; diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java index 1030b52db8..fc57956b1f 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java @@ -507,11 +507,9 @@ SerializedJson getDocument(final Event event) { } } - String pipelineValue = null; - final String document = DocumentBuilder.build(event, documentRootKey, sinkContext.getTagsTargetKey(), sinkContext.getIncludeKeys(), sinkContext.getExcludeKeys()); - return SerializedJson.fromStringAndOptionals(document, docId, routingValue, pipelineValue); + return SerializedJson.fromStringAndOptionals(document, docId, routingValue, null); } private void flushBatch(AccumulatingBulkRequest accumulatingBulkRequest) { diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java index 7d82abb916..eb37a84d81 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java @@ -215,7 +215,6 @@ void doOutput_with_invalid_version_expression_catches_NumberFormatException_and_ final EventHandle eventHandle = mock(EventHandle.class); when(event.getEventHandle()).thenReturn(eventHandle); final String index = UUID.randomUUID().toString(); - when(event.formatString(pipelineDescription.getPipelineName(), expressionEvaluator)).thenReturn(null); when(event.formatString(versionExpression, expressionEvaluator)).thenReturn("not_a_number"); when(event.formatString(indexConfiguration.getIndexAlias(), expressionEvaluator)).thenReturn(index); final Record eventRecord = new Record<>(event); @@ -292,24 +291,6 @@ void test_routing_in_document() throws IOException { assertThat(objectUnderTest2.getDocument(event).getRoutingField(), equalTo(Optional.of(routingValue))); } - @Test - void test_pipeline_in_document() throws IOException { - String pipelineValue = UUID.randomUUID().toString(); - String pipelineKey = UUID.randomUUID().toString(); - String pipelineName = UUID.randomUUID().toString(); - when(pipelineDescription.getPipelineName()).thenReturn(pipelineName); - final OpenSearchSink objectUnderTest = createObjectUnderTest(); - final Event event = JacksonEvent.builder() - .withEventType("event") - .withData(Collections.singletonMap(pipelineKey, pipelineValue)) - .build(); - assertThat(objectUnderTest.getDocument(event).getPipelineField(), equalTo(Optional.of(pipelineName))); - - when(pipelineDescription.getPipelineName()).thenReturn("${"+pipelineKey+"}"); - final OpenSearchSink objectUnderTest2 = createObjectUnderTest(); - assertThat(objectUnderTest2.getDocument(event).getPipelineField(), equalTo(Optional.of(pipelineValue))); - } - @Test void doOutput_with_invalid_version_expression_result_catches_RuntimeException_and_creates_DLQObject() throws IOException { @@ -323,7 +304,6 @@ void doOutput_with_invalid_version_expression_result_catches_RuntimeException_an final EventHandle eventHandle = mock(EventHandle.class); when(event.getEventHandle()).thenReturn(eventHandle); final String index = UUID.randomUUID().toString(); - when(event.formatString(pipelineDescription.getPipelineName(), expressionEvaluator)).thenReturn(null); when(event.formatString(versionExpression, expressionEvaluator)).thenThrow(RuntimeException.class); when(event.formatString(indexConfiguration.getIndexAlias(), expressionEvaluator)).thenReturn(index); final Record eventRecord = new Record<>(event); From d42bb1236e12889f49d73d7ff14d2b14a4926bd5 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Mon, 23 Dec 2024 11:18:13 -0800 Subject: [PATCH 33/62] Integration test changes Signed-off-by: Maxwell Brown --- .../sink/opensearch/OpenSearchSinkIT.java | 38 ++++++++++++------- .../opensearch/index/IndexConfiguration.java | 2 +- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index 49879c0c4d..f4d0c5daf0 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -826,10 +826,11 @@ public void testOutputCustomIndex() throws IOException, InterruptedException { final String testTemplateFile = Objects.requireNonNull( getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE)).getFile(); final String testIdField = "someId"; + final String testDocumentID = "${/someId}"; final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); - metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField); + metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); @@ -853,10 +854,11 @@ public void testOpenSearchBulkActionsCreate() throws IOException, InterruptedExc final String testTemplateFile = Objects.requireNonNull( getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE)).getFile(); final String testIdField = "someId"; + final String testDocumentID = "${/someId}"; final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); - metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField); + metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID); metadata.put(IndexConfiguration.ACTION, OpenSearchBulkActions.CREATE.toString()); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); @@ -881,10 +883,11 @@ public void testOpenSearchBulkActionsCreateWithExpression() throws IOException, final String testTemplateFile = Objects.requireNonNull( getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE)).getFile(); final String testIdField = "someId"; + final String testDocumentID = "${/someId}"; final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); - metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField); + metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID); Event event = (Event) testRecords.get(0).getData(); event.getMetadata().setAttribute("action", "create"); final String actionFormatExpression = "${getMetadata(\"action\")}"; @@ -915,10 +918,11 @@ public void testOpenSearchBulkActionsCreateWithInvalidExpression() throws IOExce final String testTemplateFile = Objects.requireNonNull( getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE)).getFile(); final String testIdField = "someId"; + final String testDocumentID = "${/someId}"; final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); - metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField); + metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID); Event event = (Event) testRecords.get(0).getData(); event.getMetadata().setAttribute("action", "unknown"); final String actionFormatExpression = "${getMetadata(\"action\")}"; @@ -941,11 +945,12 @@ public void testBulkActionCreateWithActions() throws IOException, InterruptedExc final String testTemplateFile = Objects.requireNonNull( getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE)).getFile(); final String testIdField = "someId"; + final String testDocumentID = "${/someId}"; final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); - metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField); + metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.CREATE.toString()); @@ -975,11 +980,12 @@ public void testBulkActionUpdateWithActions() throws IOException, InterruptedExc getClass().getClassLoader().getResource(TEST_TEMPLATE_BULK_FILE)).getFile(); final String testIdField = "someId"; + final String testDocumentID = "${/someId}"; final String testId = "foo"; List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson2(testIdField, testId, "name", "value1"))); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); - metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField); + metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.CREATE.toString()); @@ -1025,6 +1031,7 @@ public void testBulkActionUpdateWithDocumentRootKey() throws IOException, Interr getClass().getClassLoader().getResource(TEST_TEMPLATE_BULK_FILE)).getFile(); final String testIdField = "someId"; + final String testDocumentID = "${/someId}"; final String testId = "foo"; final String documentRootKey = "root_key"; @@ -1039,7 +1046,7 @@ public void testBulkActionUpdateWithDocumentRootKey() throws IOException, Interr Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); metadata.put(IndexConfiguration.DOCUMENT_ROOT_KEY, documentRootKey); - metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField); + metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID); List> aList = new ArrayList<>(); Map actionMap = new HashMap<>(); actionMap.put("type", OpenSearchBulkActions.CREATE.toString()); @@ -1093,6 +1100,7 @@ public void testBulkActionUpsertWithActionsAndNoCreate() throws IOException, Int getClass().getClassLoader().getResource(TEST_TEMPLATE_BULK_FILE)).getFile(); final String testIdField = "someId"; + final String testDocumentID = "${/someId}"; final String testId = "foo"; List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson2(testIdField, testId, "key", "value"))); @@ -1102,7 +1110,7 @@ public void testBulkActionUpsertWithActionsAndNoCreate() throws IOException, Int aList.add(actionMap); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); - metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField); + metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID); metadata.put(IndexConfiguration.ACTIONS, aList); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); @@ -1125,11 +1133,12 @@ public void testBulkActionUpsertWithActions() throws IOException, InterruptedExc getClass().getClassLoader().getResource(TEST_TEMPLATE_BULK_FILE)).getFile(); final String testIdField = "someId"; + final String testDocumentID = "${/someId}"; final String testId = "foo"; List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson2(testIdField, testId, "name", "value1"))); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); - metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField); + metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.CREATE.toString()); @@ -1177,10 +1186,11 @@ public void testBulkActionUpsertWithoutCreate() throws IOException, InterruptedE getClass().getClassLoader().getResource(TEST_TEMPLATE_BULK_FILE)).getFile(); final String testIdField = "someId"; + final String testDocumentID = "${/someId}"; final String testId = "foo"; List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson3(testIdField, testId, "name", "value1", "newKey", "newValue"))); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); - metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField); + metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.UPSERT.toString()); @@ -1213,11 +1223,12 @@ public void testBulkActionDeleteWithActions() throws IOException, InterruptedExc getClass().getClassLoader().getResource(TEST_TEMPLATE_BULK_FILE)).getFile(); final String testIdField = "someId"; + final String testDocumentID = "${/someId}"; final String testId = "foo"; List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); - metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField); + metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.DELETE.toString()); @@ -1315,7 +1326,7 @@ public void testEventOutputWithSpecialAndExtremeValues(final Object testValue) t } @ParameterizedTest - @ValueSource(strings = {"info/ids/id", "id"}) + @ValueSource(strings = {"${/info/ids/id}", "${/id}"}) public void testOpenSearchDocumentId(final String testDocumentIdField) throws IOException, InterruptedException { final String expectedId = UUID.randomUUID().toString(); final String testIndexAlias = "test_index"; @@ -1640,6 +1651,7 @@ public void testOutputManagementDisabled() throws IOException, InterruptedExcept securityAccessor.createUser(username, password, roleName); final String testIdField = "someId"; + final String testDocumentID = "${/someId}"; final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); @@ -1648,7 +1660,7 @@ public void testOutputManagementDisabled() throws IOException, InterruptedExcept metadata.put(IndexConfiguration.INDEX_TYPE, IndexType.MANAGEMENT_DISABLED.getValue()); metadata.put(USERNAME, username); metadata.put(PASSWORD, password); - metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField); + metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java index ebd1240655..b44fdc1076 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java @@ -55,7 +55,7 @@ public class IndexConfiguration { public static final String MAX_LOCAL_COMPRESSIONS_FOR_ESTIMATION = "max_local_compressions_for_estimation"; public static final String FLUSH_TIMEOUT = "flush_timeout"; public static final String DOCUMENT_ID = "document_id"; - public static final String ROUTING_FIELD = "routing_field"; + public static final String ROUTING_FIELD = "routing"; public static final String ROUTING = "routing"; public static final String PIPELINE = "pipeline"; public static final String ISM_POLICY_FILE = "ism_policy_file"; From 44f852985a2acbc4e36502ffadec6326b19ded18 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Mon, 23 Dec 2024 11:44:53 -0800 Subject: [PATCH 34/62] integration test adjustments Signed-off-by: Maxwell Brown --- .../sink/opensearch/OpenSearchSinkIT.java | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index f4d0c5daf0..640ededc53 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -889,12 +889,7 @@ public void testOpenSearchBulkActionsCreateWithExpression() throws IOException, Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID); Event event = (Event) testRecords.get(0).getData(); - event.getMetadata().setAttribute("action", "create"); - final String actionFormatExpression = "${getMetadata(\"action\")}"; - when(expressionEvaluator.isValidFormatExpression(actionFormatExpression)).thenReturn(true); - when(expressionEvaluator.isValidExpressionStatement("getMetadata(\"action\")")).thenReturn(true); - when(expressionEvaluator.evaluate("getMetadata(\"action\")", event)).thenReturn(event.getMetadata().getAttribute("action")); - metadata.put(IndexConfiguration.ACTION, actionFormatExpression); + event.getMetadata().setAttribute("action", "create");metadata.put(IndexConfiguration.ACTION, "create"); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); @@ -924,12 +919,7 @@ public void testOpenSearchBulkActionsCreateWithInvalidExpression() throws IOExce Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID); Event event = (Event) testRecords.get(0).getData(); - event.getMetadata().setAttribute("action", "unknown"); - final String actionFormatExpression = "${getMetadata(\"action\")}"; - when(expressionEvaluator.isValidFormatExpression(actionFormatExpression)).thenReturn(true); - when(expressionEvaluator.isValidExpressionStatement("getMetadata(\"action\")")).thenReturn(true); - when(expressionEvaluator.evaluate("getMetadata(\"action\")", event)).thenReturn(event.getMetadata().getAttribute("action")); - metadata.put(IndexConfiguration.ACTION, actionFormatExpression); + event.getMetadata().setAttribute("action", "unknown");metadata.put(IndexConfiguration.ACTION, "unknown"); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); @@ -1326,7 +1316,7 @@ public void testEventOutputWithSpecialAndExtremeValues(final Object testValue) t } @ParameterizedTest - @ValueSource(strings = {"${/info/ids/id}", "${/id}"}) + @ValueSource(strings = {"info/ids/id", "id"}) public void testOpenSearchDocumentId(final String testDocumentIdField) throws IOException, InterruptedException { final String expectedId = UUID.randomUUID().toString(); final String testIndexAlias = "test_index"; @@ -1335,11 +1325,12 @@ public void testOpenSearchDocumentId(final String testDocumentIdField) throws IO .withEventType("event") .build(); testEvent.put(testDocumentIdField, expectedId); + String testDocumentId = "${/" + testDocumentIdField + "}"; final List> testRecords = Collections.singletonList(new Record<>(testEvent)); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, null); - metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentIdField); + metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentId); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); From 8fa8501bbe2b763ff4cf255844f91c050b5b98bd Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Mon, 23 Dec 2024 11:59:26 -0800 Subject: [PATCH 35/62] remove Routing Field since its not in documentation Signed-off-by: Maxwell Brown --- .../sink/opensearch/OpenSearchSinkIT.java | 8 +++----- .../plugins/sink/opensearch/OpenSearchSink.java | 6 +----- .../opensearch/index/IndexConfiguration.java | 12 ------------ .../sink/opensearch/OpenSearchSinkTest.java | 17 ----------------- 4 files changed, 4 insertions(+), 39 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index 640ededc53..0d5d672938 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -89,6 +89,7 @@ import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.closeTo; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.params.provider.Arguments.arguments; import static org.mockito.ArgumentMatchers.any; @@ -908,7 +909,7 @@ public void testOpenSearchBulkActionsCreateWithExpression() throws IOException, } @Test - public void testOpenSearchBulkActionsCreateWithInvalidExpression() throws IOException, InterruptedException { + public void testOpenSearchBulkActionsCreateWithInvalidExpression() throws IOException { final String testIndexAlias = "test-alias"; final String testTemplateFile = Objects.requireNonNull( getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE)).getFile(); @@ -921,12 +922,9 @@ public void testOpenSearchBulkActionsCreateWithInvalidExpression() throws IOExce Event event = (Event) testRecords.get(0).getData(); event.getMetadata().setAttribute("action", "unknown");metadata.put(IndexConfiguration.ACTION, "unknown"); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); - final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); - sink.output(testRecords); + assertThrows(IllegalArgumentException.class, () -> createObjectUnderTest(openSearchSinkConfig, true)); final List> retSources = getSearchResponseDocSources(testIndexAlias); assertThat(retSources.size(), equalTo(0)); - assertThat(sink.getInvalidActionErrorsCount(), equalTo(1.0)); - sink.shutdown(); } @Test diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java index fc57956b1f..0d35ac6c05 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java @@ -114,7 +114,6 @@ public class OpenSearchSink extends AbstractSink> { private final IndexType indexType; private final String documentIdField; private final String documentId; - private final String routingField; private final String routing; private final String pipeline; private final String action; @@ -171,7 +170,6 @@ public OpenSearchSink(final PluginSetting pluginSetting, this.indexType = openSearchSinkConfig.getIndexConfiguration().getIndexType(); this.documentIdField = openSearchSinkConfig.getIndexConfiguration().getDocumentIdField(); this.documentId = openSearchSinkConfig.getIndexConfiguration().getDocumentId(); - this.routingField = openSearchSinkConfig.getIndexConfiguration().getRoutingField(); this.routing = openSearchSinkConfig.getIndexConfiguration().getRouting(); this.action = openSearchSinkConfig.getIndexConfiguration().getAction(); this.actions = openSearchSinkConfig.getIndexConfiguration().getActions(); @@ -497,9 +495,7 @@ SerializedJson getDocument(final Event event) { } String routingValue = null; - if (routingField != null) { - routingValue = event.get(routingField, String.class); - } else if (routing != null) { + if (routing != null) { try { routingValue = event.formatString(routing, expressionEvaluator); } catch (final ExpressionEvaluationException | EventKeyNotFoundException e) { diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java index b44fdc1076..a80c9125d6 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java @@ -77,7 +77,6 @@ public class IndexConfiguration { private final Map indexTemplate; private final String documentIdField; private final String documentId; - private final String routingField; private final String routing; private final long bulkSize; private final boolean estimateBulkSizeUsingCompression; @@ -141,7 +140,6 @@ private IndexConfiguration(final Builder builder) { this.estimateBulkSizeUsingCompression = builder.estimateBulkSizeUsingCompression; this.maxLocalCompressionsForEstimation = builder.maxLocalCompressionsForEstimation; this.flushTimeout = builder.flushTimeout; - this.routingField = builder.routingField; this.routing = builder.routing; String documentIdField = builder.documentIdField; @@ -290,10 +288,6 @@ public String getDocumentIdField() { public String getDocumentId() { return documentId; } - public String getRoutingField() { - return routingField; - } - public String getRouting() { return routing; } @@ -423,7 +417,6 @@ public static class Builder { private String templateContent; private int numShards; private int numReplicas; - private String routingField; private String routing; private String pipeline; private String documentIdField; @@ -491,11 +484,6 @@ public Builder withDocumentId(final String documentId) { return this; } - public Builder withRoutingField(final String routingField) { - this.routingField = routingField; - return this; - } - public Builder withRouting(final String routing) { this.routing = routing; return this; diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java index eb37a84d81..79ebb20f01 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java @@ -146,7 +146,6 @@ void setup() { when(indexConfiguration.getAction()).thenReturn("index"); when(indexConfiguration.getDocumentId()).thenReturn(null); when(indexConfiguration.getDocumentIdField()).thenReturn(null); - when(indexConfiguration.getRoutingField()).thenReturn(null); when(indexConfiguration.getRouting()).thenReturn(null); when(indexConfiguration.getActions()).thenReturn(null); when(indexConfiguration.getDocumentRootKey()).thenReturn(null); @@ -259,22 +258,6 @@ void doOutput_with_invalid_version_expression_catches_NumberFormatException_and_ verify(dynamicDocumentVersionDroppedEvents).increment(); } - @Test - void test_routing_field_in_document() throws IOException { - String routingFieldKey = UUID.randomUUID().toString(); - String routingKey = UUID.randomUUID().toString(); - String routingFieldValue = UUID.randomUUID().toString(); - when(indexConfiguration.getRoutingField()).thenReturn(routingFieldKey); - when(indexConfiguration.getRouting()).thenReturn(routingKey); - final OpenSearchSink objectUnderTest = createObjectUnderTest(); - final Event event = JacksonEvent.builder() - .withEventType("event") - .withData(Collections.singletonMap(routingFieldKey, routingFieldValue)) - .build(); - assertThat(objectUnderTest.getDocument(event).getRoutingField(), equalTo(Optional.of(routingFieldValue))); - - } - @Test void test_routing_in_document() throws IOException { String routingValue = UUID.randomUUID().toString(); From b0aaee27bc10d6992da57dd330a642e44ef29c2c Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Mon, 23 Dec 2024 12:05:04 -0800 Subject: [PATCH 36/62] remove routing field integration test Signed-off-by: Maxwell Brown --- .../sink/opensearch/OpenSearchSinkIT.java | 26 ------------------- .../opensearch/index/IndexConfiguration.java | 1 - 2 files changed, 27 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index 0d5d672938..0ac74d5975 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -1340,32 +1340,6 @@ public void testOpenSearchDocumentId(final String testDocumentIdField) throws IO sink.shutdown(); } - @ParameterizedTest - @ValueSource(strings = {"info/ids/rid", "rid"}) - public void testOpenSearchRoutingField(final String testRoutingField) throws IOException, InterruptedException { - final String expectedRoutingField = UUID.randomUUID().toString(); - final String testIndexAlias = "test_index"; - final Event testEvent = JacksonEvent.builder() - .withData(Map.of("arbitrary_data", UUID.randomUUID().toString())) - .withEventType("event") - .build(); - testEvent.put(testRoutingField, expectedRoutingField); - - final List> testRecords = Collections.singletonList(new Record<>(testEvent)); - - Map metadata = initializeConfigurationMetadata(null, testIndexAlias, null); - metadata.put(IndexConfiguration.ROUTING_FIELD, testRoutingField); - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); - final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); - sink.output(testRecords); - - final List routingFields = getSearchResponseRoutingFields(testIndexAlias); - for (String routingField : routingFields) { - assertThat(routingField, equalTo(expectedRoutingField)); - } - sink.shutdown(); - } - @ParameterizedTest @ValueSource(strings = {"", "info/ids/rid", "rid"}) public void testOpenSearchRouting(final String testRouting) throws IOException, InterruptedException { diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java index a80c9125d6..8a2d1ffe01 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java @@ -55,7 +55,6 @@ public class IndexConfiguration { public static final String MAX_LOCAL_COMPRESSIONS_FOR_ESTIMATION = "max_local_compressions_for_estimation"; public static final String FLUSH_TIMEOUT = "flush_timeout"; public static final String DOCUMENT_ID = "document_id"; - public static final String ROUTING_FIELD = "routing"; public static final String ROUTING = "routing"; public static final String PIPELINE = "pipeline"; public static final String ISM_POLICY_FILE = "ism_policy_file"; From 507a9358083e906b785ead6af49d0e861062f2da Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Mon, 23 Dec 2024 12:08:58 -0800 Subject: [PATCH 37/62] integration test adjustment :/ Signed-off-by: Maxwell Brown --- .../dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index 0ac74d5975..8d6576a95d 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -919,8 +919,7 @@ public void testOpenSearchBulkActionsCreateWithInvalidExpression() throws IOExce final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID); - Event event = (Event) testRecords.get(0).getData(); - event.getMetadata().setAttribute("action", "unknown");metadata.put(IndexConfiguration.ACTION, "unknown"); + metadata.put(IndexConfiguration.ACTION, "unknown"); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); assertThrows(IllegalArgumentException.class, () -> createObjectUnderTest(openSearchSinkConfig, true)); final List> retSources = getSearchResponseDocSources(testIndexAlias); From f6081adbaf9f01fc1f83bfc759249302a29914f6 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Mon, 23 Dec 2024 12:17:13 -0800 Subject: [PATCH 38/62] fix invalidactions integration test Signed-off-by: Maxwell Brown --- .../dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index 8d6576a95d..cd6456363a 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -922,8 +922,6 @@ public void testOpenSearchBulkActionsCreateWithInvalidExpression() throws IOExce metadata.put(IndexConfiguration.ACTION, "unknown"); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); assertThrows(IllegalArgumentException.class, () -> createObjectUnderTest(openSearchSinkConfig, true)); - final List> retSources = getSearchResponseDocSources(testIndexAlias); - assertThat(retSources.size(), equalTo(0)); } @Test From b8d1b037246dd1e177fd468a751487ab3199a06f Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Thu, 2 Jan 2025 15:59:59 -0800 Subject: [PATCH 39/62] addressing various pr comments Signed-off-by: Maxwell Brown --- .../sink/opensearch/OpenSearchSinkIT.java | 1 + .../sink/opensearch/BulkRetryStrategy.java | 6 -- .../opensearch/ConnectionConfiguration.java | 7 +- .../opensearch/OpenSearchClientRefresher.java | 8 +-- .../sink/opensearch/OpenSearchSink.java | 1 + .../OpenSearchSinkConfiguration.java | 2 +- .../sink/opensearch/RetryConfiguration.java | 1 + .../AwsAuthenticationConfiguration.java | 4 +- .../OpenSearchSinkConfig.java | 71 ++++++++----------- .../opensearch/index/IndexConfiguration.java | 12 ++-- .../ConnectionConfigurationTests.java | 1 + .../OpenSearchClientRefresherTest.java | 1 + .../OpenSearchSinkConfigurationTests.java | 1 + .../sink/opensearch/OpenSearchSinkTest.java | 1 + .../opensearch/RetryConfigurationTests.java | 1 + .../index/IndexConfigurationTests.java | 2 +- 16 files changed, 53 insertions(+), 67 deletions(-) rename data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/{ => configuration}/OpenSearchSinkConfig.java (66%) diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index cd6456363a..8836031412 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -47,6 +47,7 @@ import org.opensearch.dataprepper.model.plugin.PluginConfigObservable; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.sink.SinkContext; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig; import org.opensearch.dataprepper.plugins.sink.opensearch.index.AbstractIndexManager; import org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration; import org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConstants; diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategy.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategy.java index 9deea290e6..f4ff8acafd 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategy.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategy.java @@ -104,9 +104,6 @@ public final class BulkRetryStrategy { private final PluginMetrics pluginMetrics; private final Supplier bulkRequestSupplier; private final int maxRetries; - private final String pluginId; - private final String pluginName; - private final String pipelineName; private final ObjectMapper objectMapper; private final Counter sentDocumentsCounter; @@ -156,9 +153,6 @@ public BulkRetryStrategy(final RequestFunction { +public class OpenSearchClientRefresher implements PluginComponentRefresher { static final String CREDENTIALS_CHANGED = "credentialsChanged"; static final String CLIENT_REFRESH_ERRORS = "clientRefreshErrors"; private static final Logger LOG = LoggerFactory.getLogger(OpenSearchClientRefresher.class); @@ -54,10 +54,6 @@ public OpenSearchClient get() { } @Override - public void update(final PluginSetting pluginSetting) { - //not implemented - } - public void update(OpenSearchSinkConfig openSearchSinkConfig) { final ConnectionConfiguration newConfig = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); if (basicAuthChanged(newConfig)) { diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java index 0d35ac6c05..adfc801c93 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java @@ -54,6 +54,7 @@ import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.SerializedJson; import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.ActionConfiguration; import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.DlqConfiguration; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig; import org.opensearch.dataprepper.plugins.sink.opensearch.dlq.FailedBulkOperation; import org.opensearch.dataprepper.plugins.sink.opensearch.dlq.FailedBulkOperationConverter; import org.opensearch.dataprepper.plugins.sink.opensearch.dlq.FailedDlqData; diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfiguration.java index 80d3d60528..84bf3c28de 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfiguration.java @@ -6,6 +6,7 @@ package org.opensearch.dataprepper.plugins.sink.opensearch; import org.opensearch.dataprepper.expression.ExpressionEvaluator; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig; import org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration; import static com.google.common.base.Preconditions.checkNotNull; @@ -44,7 +45,6 @@ public static OpenSearchSinkConfiguration readOSConfig(final OpenSearchSinkConfi } public static OpenSearchSinkConfiguration readOSConfig(final OpenSearchSinkConfig openSearchSinkConfig, final ExpressionEvaluator expressionEvaluator) { - openSearchSinkConfig.validateConfig(); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig, expressionEvaluator); diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfiguration.java index 666a2dd543..a3c45d6a2d 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfiguration.java @@ -6,6 +6,7 @@ package org.opensearch.dataprepper.plugins.sink.opensearch; import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.DlqConfiguration; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig; import java.util.Optional; diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AwsAuthenticationConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AwsAuthenticationConfiguration.java index b3d428b96f..726aaebb73 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AwsAuthenticationConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AwsAuthenticationConfiguration.java @@ -31,7 +31,7 @@ public class AwsAuthenticationConfiguration { private Map awsStsHeaderOverrides; @JsonProperty("serverless") - private Boolean serverless = false; + private boolean serverless = false; @JsonProperty("serverless_options") private ServerlessOptions serverlessOptions; @@ -52,7 +52,7 @@ public Map getAwsStsHeaderOverrides() { return awsStsHeaderOverrides; } - public Boolean isServerlessCollection() { + public boolean isServerlessCollection() { return serverless; } diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfig.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java similarity index 66% rename from data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfig.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java index 785024a68a..1ebda5749e 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfig.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java @@ -1,19 +1,24 @@ -package org.opensearch.dataprepper.plugins.sink.opensearch; +package org.opensearch.dataprepper.plugins.sink.opensearch.configuration; import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.validation.Valid; +import jakarta.validation.constraints.AssertTrue; import lombok.Getter; import org.apache.commons.lang3.EnumUtils; import org.opensearch.dataprepper.model.opensearch.OpenSearchBulkActions; -import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.ActionConfiguration; -import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.AwsAuthenticationConfiguration; -import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.DlqConfiguration; +import org.opensearch.dataprepper.plugins.sink.opensearch.DistributionVersion; import org.opensearch.dataprepper.plugins.sink.opensearch.index.TemplateType; import java.util.List; import java.util.Objects; +import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.DEFAULT_BULK_SIZE; +import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.DEFAULT_ESTIMATE_BULK_SIZE_USING_COMPRESSION; +import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.DEFAULT_FLUSH_TIMEOUT; +import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.DEFAULT_MAX_LOCAL_COMPRESSIONS_FOR_ESTIMATION; + public class OpenSearchSinkConfig { + @Getter @JsonProperty("hosts") private List hosts; @@ -45,7 +50,7 @@ public class OpenSearchSinkConfig { @Getter @JsonProperty("insecure") - private Boolean insecure = false; + private boolean insecure = false; @Getter @JsonProperty("proxy") @@ -58,8 +63,9 @@ public class OpenSearchSinkConfig { @JsonProperty("enable_request_compression") private Boolean enableRequestCompression = null; - public Boolean getEnableRequestCompression(boolean defaultValue) { - return Objects.requireNonNullElse(enableRequestCompression, defaultValue); + public boolean getEnableRequestCompression() { + final DistributionVersion distributionVersion = DistributionVersion.fromTypeName(getDistributionVersion()); + return Objects.requireNonNullElse(enableRequestCompression, !DistributionVersion.ES6.equals(distributionVersion)); } @Getter @@ -90,33 +96,21 @@ public Boolean getEnableRequestCompression(boolean defaultValue) { @JsonProperty("number_of_replicas") private Integer numReplicas = 0; + @Getter @JsonProperty("bulk_size") - private Long bulkSize; - - public Long getBulkSize(Long defaultBulkSize) { - return bulkSize == null ? defaultBulkSize : bulkSize; - } + private Long bulkSize = DEFAULT_BULK_SIZE; + @Getter @JsonProperty("estimate_bulk_size_using_compression") - private Boolean estimateBulkSizeUsingCompression; - - public Boolean getEstimateBulkSizeUsingCompression(boolean defaultValue) { - return Objects.requireNonNullElse(estimateBulkSizeUsingCompression, defaultValue); - } + private boolean estimateBulkSizeUsingCompression = DEFAULT_ESTIMATE_BULK_SIZE_USING_COMPRESSION; + @Getter @JsonProperty("max_local_compressions_for_estimation") - private Integer maxLocalCompressionsForEstimation; - - public Integer getMaxLocalCompressionsForEstimation(Integer defaultValue) { - return Objects.requireNonNullElse(maxLocalCompressionsForEstimation, defaultValue); - } + private Integer maxLocalCompressionsForEstimation = DEFAULT_MAX_LOCAL_COMPRESSIONS_FOR_ESTIMATION; + @Getter @JsonProperty("flush_timeout") - private Long flushTimeout = null; - - public Long getFlushTimeout(Long defaultFlushTimeout) { - return flushTimeout == null ? defaultFlushTimeout : flushTimeout; - } + private Long flushTimeout = DEFAULT_FLUSH_TIMEOUT; @Getter @JsonProperty("document_version_type") @@ -128,7 +122,7 @@ public Long getFlushTimeout(Long defaultFlushTimeout) { @Getter @JsonProperty("normalize_index") - private Boolean normalizeIndex = false; + private boolean normalizeIndex = false; @Getter @JsonProperty("document_id") @@ -167,27 +161,22 @@ public Long getFlushTimeout(Long defaultFlushTimeout) { @JsonProperty("dlq") private DlqConfiguration dlq; - - public void validateConfig() { - isActionValid(); - isDlqValid(); - } - - void isDlqValid() { + @AssertTrue(message = "dlq_file option cannot be used along with dlq option") + public boolean isDlqValid() { if (dlq != null) { if (dlqFile!= null) { - throw new IllegalArgumentException("dlq_file option cannot be used along with dlq option"); + return false; } } + return true; } - void isActionValid() { + @AssertTrue(message = "action must be one of [index, create, update, upsert, delete]") + public boolean isActionValid() { if (EnumUtils.isValidEnumIgnoreCase(OpenSearchBulkActions.class, action)) { - return; + return true; } - - System.out.println(action); - throw new IllegalArgumentException("action must be one of [index, create, update, upsert, delete]"); + return false; } } diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java index 8a2d1ffe01..1374b04841 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java @@ -13,7 +13,7 @@ import org.opensearch.dataprepper.model.opensearch.OpenSearchBulkActions; import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException; import org.opensearch.dataprepper.plugins.sink.opensearch.DistributionVersion; -import org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchSinkConfig; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig; import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.ActionConfiguration; import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.AwsAuthenticationConfiguration; import org.opensearch.dataprepper.plugins.sink.opensearch.s3.FileReader; @@ -218,12 +218,12 @@ public static IndexConfiguration readIndexConfig(final OpenSearchSinkConfig open builder = builder.withNumShards(openSearchSinkConfig.getNumShards()) .withNumReplicas(openSearchSinkConfig.getNumReplicas()) - .withBulkSize(openSearchSinkConfig.getBulkSize(DEFAULT_BULK_SIZE)) - .withEstimateBulkSizeUsingCompression(openSearchSinkConfig.getEstimateBulkSizeUsingCompression(DEFAULT_ESTIMATE_BULK_SIZE_USING_COMPRESSION)) - .withMaxLocalCompressionsForEstimation(openSearchSinkConfig.getMaxLocalCompressionsForEstimation(DEFAULT_MAX_LOCAL_COMPRESSIONS_FOR_ESTIMATION)) - .withFlushTimeout(openSearchSinkConfig.getFlushTimeout(DEFAULT_FLUSH_TIMEOUT)) + .withBulkSize(openSearchSinkConfig.getBulkSize()) + .withEstimateBulkSizeUsingCompression(openSearchSinkConfig.isEstimateBulkSizeUsingCompression()) + .withMaxLocalCompressionsForEstimation(openSearchSinkConfig.getMaxLocalCompressionsForEstimation()) + .withFlushTimeout(openSearchSinkConfig.getFlushTimeout()) .withVersionType(openSearchSinkConfig.getVersionType()) - .withNormalizeIndex(openSearchSinkConfig.getNormalizeIndex()) + .withNormalizeIndex(openSearchSinkConfig.isNormalizeIndex()) .withIsmPolicyFile(openSearchSinkConfig.getIsmPolicyFile()) .withDocumentRootKey(openSearchSinkConfig.getDocumentRootKey()) .withDistributionVersion(openSearchSinkConfig.getDistributionVersion()); diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java index dc68864a0b..ffb34f9397 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java @@ -20,6 +20,7 @@ import org.opensearch.dataprepper.aws.api.AwsCredentialsOptions; import org.opensearch.dataprepper.aws.api.AwsCredentialsSupplier; import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.PreSerializedJsonpMapper; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.http.apache.ApacheHttpClient; import software.amazon.awssdk.regions.Region; diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresherTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresherTest.java index 9bbed3a456..24c27bed89 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresherTest.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresherTest.java @@ -9,6 +9,7 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.client.opensearch.OpenSearchClient; import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig; import org.opensearch.dataprepper.plugins.source.opensearch.AuthConfig; import java.util.function.Function; diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfigurationTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfigurationTests.java index 12acd8053a..cfe97dbbbe 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfigurationTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfigurationTests.java @@ -10,6 +10,7 @@ import org.junit.Test; import org.opensearch.dataprepper.expression.ExpressionEvaluator; import org.opensearch.dataprepper.model.opensearch.OpenSearchBulkActions; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig; import java.io.File; import java.io.IOException; diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java index 79ebb20f01..d2e94efbf5 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java @@ -31,6 +31,7 @@ import org.opensearch.dataprepper.model.plugin.PluginConfigObservable; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.sink.SinkContext; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig; import org.opensearch.dataprepper.plugins.sink.opensearch.dlq.FailedDlqData; import org.opensearch.dataprepper.plugins.sink.opensearch.index.DocumentBuilder; import org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration; diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfigurationTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfigurationTests.java index 3f8e4d11a8..4eb81de071 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfigurationTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfigurationTests.java @@ -10,6 +10,7 @@ import org.junit.Test; import org.opensearch.dataprepper.plugins.dlq.s3.S3DlqWriterConfig; import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.DlqConfiguration; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig; import java.io.File; import java.io.IOException; diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfigurationTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfigurationTests.java index 3e61ba55c0..c3c59abcd8 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfigurationTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfigurationTests.java @@ -16,7 +16,7 @@ import org.opensearch.dataprepper.expression.ExpressionEvaluator; import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException; import org.opensearch.dataprepper.plugins.sink.opensearch.DistributionVersion; -import org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchSinkConfig; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig; import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.http.AbortableInputStream; import software.amazon.awssdk.services.s3.S3Client; From 7ee34424a8858af1af5f003a674764b4b2c92945 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Thu, 2 Jan 2025 16:49:28 -0800 Subject: [PATCH 40/62] action type as enum Signed-off-by: Maxwell Brown --- .../configuration/ActionConfiguration.java | 14 +++++--------- .../configuration/OpenSearchSinkConfig.java | 15 +++++---------- 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ActionConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ActionConfiguration.java index f66b816d7a..96519fceaa 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ActionConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ActionConfiguration.java @@ -8,20 +8,16 @@ import org.opensearch.dataprepper.model.opensearch.OpenSearchBulkActions; public class ActionConfiguration { - @Getter @Size(min = 1, message = "type cannot be empty") @JsonProperty("type") - private String type; + private OpenSearchBulkActions type; + + public String getType() { + return type.toString(); + } @Getter @JsonProperty("when") private String when; - @AssertTrue(message = "type must be one of index, create, update, upsert, delete") - boolean isTypeValid() { - if (EnumUtils.isValidEnumIgnoreCase(OpenSearchBulkActions.class, type)) { - return true; - } - return false; - } } diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java index 1ebda5749e..6278ce2ba3 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java @@ -136,9 +136,12 @@ public boolean getEnableRequestCompression() { @JsonProperty("ism_policy_file") private String ismPolicyFile = null; - @Getter @JsonProperty("action") - private String action = OpenSearchBulkActions.INDEX.toString(); + private OpenSearchBulkActions action = OpenSearchBulkActions.INDEX; + + public String getAction() { + return action.toString(); + } @Getter @Valid @@ -171,13 +174,5 @@ public boolean isDlqValid() { return true; } - @AssertTrue(message = "action must be one of [index, create, update, upsert, delete]") - public boolean isActionValid() { - if (EnumUtils.isValidEnumIgnoreCase(OpenSearchBulkActions.class, action)) { - return true; - } - return false; - } - } From 544306077a4565820e117f1ab5ce02e20b2e4fc0 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Fri, 3 Jan 2025 10:20:38 -0800 Subject: [PATCH 41/62] indexconfiguration constants to opensearchsinkconfig Signed-off-by: Maxwell Brown --- .../configuration/OpenSearchSinkConfig.java | 11 ++++------- .../sink/opensearch/index/IndexConfiguration.java | 8 ++++---- .../plugins/sink/opensearch/OpenSearchSinkTest.java | 4 ++-- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java index 6278ce2ba3..9243fbfaca 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java @@ -4,7 +4,6 @@ import jakarta.validation.Valid; import jakarta.validation.constraints.AssertTrue; import lombok.Getter; -import org.apache.commons.lang3.EnumUtils; import org.opensearch.dataprepper.model.opensearch.OpenSearchBulkActions; import org.opensearch.dataprepper.plugins.sink.opensearch.DistributionVersion; import org.opensearch.dataprepper.plugins.sink.opensearch.index.TemplateType; @@ -12,13 +11,11 @@ import java.util.List; import java.util.Objects; -import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.DEFAULT_BULK_SIZE; -import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.DEFAULT_ESTIMATE_BULK_SIZE_USING_COMPRESSION; -import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.DEFAULT_FLUSH_TIMEOUT; -import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.DEFAULT_MAX_LOCAL_COMPRESSIONS_FOR_ESTIMATION; - public class OpenSearchSinkConfig { - + public static final long DEFAULT_BULK_SIZE = 5L; + public static final boolean DEFAULT_ESTIMATE_BULK_SIZE_USING_COMPRESSION = false; + public static final int DEFAULT_MAX_LOCAL_COMPRESSIONS_FOR_ESTIMATION = 2; + public static final long DEFAULT_FLUSH_TIMEOUT = 60_000L; @Getter @JsonProperty("hosts") private List hosts; diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java index 1374b04841..0f399fbfde 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java @@ -37,6 +37,10 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import static org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig.DEFAULT_BULK_SIZE; +import static org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig.DEFAULT_ESTIMATE_BULK_SIZE_USING_COMPRESSION; +import static org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig.DEFAULT_FLUSH_TIMEOUT; +import static org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig.DEFAULT_MAX_LOCAL_COMPRESSIONS_FOR_ESTIMATION; public class IndexConfiguration { private static final Logger LOG = LoggerFactory.getLogger(IndexConfiguration.class); @@ -58,10 +62,6 @@ public class IndexConfiguration { public static final String ROUTING = "routing"; public static final String PIPELINE = "pipeline"; public static final String ISM_POLICY_FILE = "ism_policy_file"; - public static final long DEFAULT_BULK_SIZE = 5L; - public static final boolean DEFAULT_ESTIMATE_BULK_SIZE_USING_COMPRESSION = false; - public static final int DEFAULT_MAX_LOCAL_COMPRESSIONS_FOR_ESTIMATION = 2; - public static final long DEFAULT_FLUSH_TIMEOUT = 60_000L; public static final String ACTION = "action"; public static final String ACTIONS = "actions"; public static final String SERVERLESS = "serverless"; diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java index d2e94efbf5..815698a123 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java @@ -68,8 +68,8 @@ import static org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchSink.DYNAMIC_INDEX_DROPPED_EVENTS; import static org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchSink.INVALID_ACTION_ERRORS; import static org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchSink.INVALID_VERSION_EXPRESSION_DROPPED_EVENTS; -import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.DEFAULT_BULK_SIZE; -import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.DEFAULT_FLUSH_TIMEOUT; +import static org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig.DEFAULT_BULK_SIZE; +import static org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig.DEFAULT_FLUSH_TIMEOUT; @ExtendWith(MockitoExtension.class) public class OpenSearchSinkTest { From a0de738d2ef200f8d097cad3b60eb76afc7e349a Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Fri, 3 Jan 2025 11:59:12 -0800 Subject: [PATCH 42/62] readd deprecated fields Signed-off-by: Maxwell Brown --- .../opensearch/ConnectionConfiguration.java | 19 ++- .../sink/opensearch/OpenSearchSink.java | 6 +- .../AwsAuthenticationConfiguration.java | 1 - .../configuration/OpenSearchSinkConfig.java | 125 ++++++++++++++++++ .../opensearch/index/IndexConfiguration.java | 27 +++- 5 files changed, 171 insertions(+), 7 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java index 8e2d9d1688..a5b13a68fa 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java @@ -31,8 +31,9 @@ import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.PreSerializedJsonpMapper; import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.AwsAuthenticationConfiguration; import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.ServerlessOptions; + import org.opensearch.dataprepper.plugins.source.opensearch.AuthConfig; -import org.opensearch.dataprepper.plugins.source.opensearch.configuration.ServerlessOptions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.arns.Arn; @@ -209,6 +210,7 @@ public static ConnectionConfiguration readConnectionConfiguration(final OpenSear builder.withAwsSigv4(false); final AwsAuthenticationConfiguration awsAuthenticationConfiguration = openSearchSinkConfig.getAwsAuthenticationOptions(); + boolean awsSigv4 = openSearchSinkConfig.isAwsSigv4(); if (awsAuthenticationConfiguration != null) { builder = builder.withAwsSigv4(true) .withAwsRegion(awsAuthenticationConfiguration.getAwsRegion().toString()) @@ -223,7 +225,20 @@ public static ConnectionConfiguration readConnectionConfiguration(final OpenSear .withServerlessCollectionName(serverlessOptions.getCollectionName()) .withServerlessVpceId(serverlessOptions.getVpceId()); } - } else { + } else if (awsSigv4) { + builder = builder.withAwsSigv4(awsSigv4) + .withAwsRegion(openSearchSinkConfig.getAwsRegion()) + .withAWSStsRoleArn(openSearchSinkConfig.getAwsStsRoleArn()) + .withAWSStsExternalId(openSearchSinkConfig.getAwsStsExternalId()) + .withAwsStsHeaderOverrides(openSearchSinkConfig.getAwsStsHeaderOverrides()); + + final ServerlessOptions serverlessOptions = openSearchSinkConfig.getServerlessOptions(); + if (serverlessOptions != null) { + builder = builder.withServerlessNetworkPolicyName(serverlessOptions.getNetworkPolicyName()) + .withServerlessCollectionName(serverlessOptions.getCollectionName()) + .withServerlessVpceId(serverlessOptions.getVpceId()); + } + } else { builder.withServerless(false); } diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java index adfc801c93..ab111f9b01 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java @@ -115,6 +115,7 @@ public class OpenSearchSink extends AbstractSink> { private final IndexType indexType; private final String documentIdField; private final String documentId; + private final String routingField; private final String routing; private final String pipeline; private final String action; @@ -171,6 +172,7 @@ public OpenSearchSink(final PluginSetting pluginSetting, this.indexType = openSearchSinkConfig.getIndexConfiguration().getIndexType(); this.documentIdField = openSearchSinkConfig.getIndexConfiguration().getDocumentIdField(); this.documentId = openSearchSinkConfig.getIndexConfiguration().getDocumentId(); + this.routingField = openSearchSinkConfig.getIndexConfiguration().getRoutingField(); this.routing = openSearchSinkConfig.getIndexConfiguration().getRouting(); this.action = openSearchSinkConfig.getIndexConfiguration().getAction(); this.actions = openSearchSinkConfig.getIndexConfiguration().getActions(); @@ -496,7 +498,9 @@ SerializedJson getDocument(final Event event) { } String routingValue = null; - if (routing != null) { + if (routingField != null) { + routingValue = event.get(routingField, String.class); + } else if (routing != null) { try { routingValue = event.formatString(routing, expressionEvaluator); } catch (final ExpressionEvaluationException | EventKeyNotFoundException e) { diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AwsAuthenticationConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AwsAuthenticationConfiguration.java index 726aaebb73..d99e494179 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AwsAuthenticationConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AwsAuthenticationConfiguration.java @@ -7,7 +7,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.validation.constraints.Size; -import org.opensearch.dataprepper.plugins.source.opensearch.configuration.ServerlessOptions; import software.amazon.awssdk.regions.Region; import java.util.Map; diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java index 9243fbfaca..a14fb38cd7 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java @@ -3,12 +3,14 @@ import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.validation.Valid; import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.Size; import lombok.Getter; import org.opensearch.dataprepper.model.opensearch.OpenSearchBulkActions; import org.opensearch.dataprepper.plugins.sink.opensearch.DistributionVersion; import org.opensearch.dataprepper.plugins.sink.opensearch.index.TemplateType; import java.util.List; +import java.util.Map; import java.util.Objects; public class OpenSearchSinkConfig { @@ -16,6 +18,7 @@ public class OpenSearchSinkConfig { public static final boolean DEFAULT_ESTIMATE_BULK_SIZE_USING_COMPRESSION = false; public static final int DEFAULT_MAX_LOCAL_COMPRESSIONS_FOR_ESTIMATION = 2; public static final long DEFAULT_FLUSH_TIMEOUT = 60_000L; + private static final String DEFAULT_AWS_REGION = "us-east-1"; @Getter @JsonProperty("hosts") private List hosts; @@ -171,5 +174,127 @@ public boolean isDlqValid() { return true; } + @Deprecated + @Getter + @JsonProperty("aws_sigv4") + private boolean awsSigv4 = false; + + @Deprecated + @AssertTrue(message = "aws_sigv4 option cannot be used along with aws option. It is preferred to only use aws option as aws_sigv4 is deprecated.") + public boolean isNotAwsSigv4AndAwsOption() { + if (awsAuthenticationOptions != null && awsSigv4) { + return false; + } + return true; + } + + @Deprecated + @Getter + @JsonProperty("aws_region") + @Size(min = 1, message = "Region cannot be empty string") + private String awsRegion = DEFAULT_AWS_REGION; + + @Deprecated + @AssertTrue(message = "aws_region option cannot be used along with aws option. It is preferred to only use aws option as aws_region is deprecated.") + public boolean isNotAwsRegionAndAwsOption() { + if (awsAuthenticationOptions != null && awsRegion != null) { + return false; + } + return true; + } + + @Deprecated + @Getter + @JsonProperty("aws_sts_role_arn") + @Size(min = 20, max = 2048, message = "awsStsRoleArn length should be between 1 and 2048 characters") + private String awsStsRoleArn = null; + + @Deprecated + @AssertTrue(message = "aws_sts_role_arn cannot be used along with aws option. It is preferred to only use aws option as aws_sts_role_arn is deprecated.") + public boolean isNotAwsStsRoleArnAndAws() { + if (awsAuthenticationOptions != null && awsStsRoleArn != null) { + return false; + } + return true; + } + + @Deprecated + @Getter + @JsonProperty("aws_sts_external_id") + @Size(min = 2, max = 1224, message = "awsStsExternalId length should be between 2 and 1224 characters") + private String awsStsExternalId = null; + + @Deprecated + @AssertTrue(message = "aws_sts_external_id cannot be used along with aws option. It is preferred to only use aws option as aws_sts_external_id is deprecated.") + public boolean isNotAwsStsExternalIdAndAws() { + if (awsAuthenticationOptions != null && awsStsExternalId != null) { + return false; + } + return true; + } + + @Deprecated + @Getter + @JsonProperty("aws_sts_header_overrides") + @Size(max = 5, message = "sts_header_overrides supports a maximum of 5 headers to override") + private Map awsStsHeaderOverrides; + + @Deprecated + @AssertTrue(message = "aws_sts_header_overrides cannot be used along with aws option. It is preferred to only use aws option as aws_sts_header_overrides is deprecated.") + public boolean isNotAwsStsHeaderOverridesAndAws() { + if (awsAuthenticationOptions != null && awsStsHeaderOverrides != null) { + return false; + } + return true; + } + + @Deprecated + @Getter + @JsonProperty("serverless") + private boolean serverless = false; + + @Deprecated + @AssertTrue(message = "serverless cannot be used along with aws option. It is preferred to only use aws option as serverless is deprecated.") + public boolean isNotServerlessAndAws() { + if (awsAuthenticationOptions != null && serverless) { + return false; + } + return true; + } + + @Deprecated + @Getter + @JsonProperty("serverless_options") + private ServerlessOptions serverlessOptions = null; + + @Deprecated + @AssertTrue(message = "serverless_options cannot be used along with aws option. It is preferred to only use aws option as serverless_options is deprecated.") + public boolean isNotServerlessOptionsAndAws() { + if (awsAuthenticationOptions != null && serverlessOptions != null) { + return false; + } + return true; + } + + @Deprecated + @Getter + @JsonProperty("document_id_field") + private String documentIdField = null; + + @Deprecated + @AssertTrue(message = "Both document_id_field and document_id cannot be used at the same time. It is preferred to only use document_id as document_id_field is deprecated.") + public boolean isNotDocumentIdFieldAndDocumentId() { + if (documentId != null && documentIdField != null) { + return false; + } + return true; + } + + @Deprecated + @Getter + @JsonProperty("routing_field") + private String routingField = null; + + } diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java index 0f399fbfde..e946323439 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java @@ -58,6 +58,7 @@ public class IndexConfiguration { public static final String ESTIMATE_BULK_SIZE_USING_COMPRESSION = "estimate_bulk_size_using_compression"; public static final String MAX_LOCAL_COMPRESSIONS_FOR_ESTIMATION = "max_local_compressions_for_estimation"; public static final String FLUSH_TIMEOUT = "flush_timeout"; + public static final String DOCUMENT_ID_FIELD = "document_id_field"; public static final String DOCUMENT_ID = "document_id"; public static final String ROUTING = "routing"; public static final String PIPELINE = "pipeline"; @@ -76,6 +77,7 @@ public class IndexConfiguration { private final Map indexTemplate; private final String documentIdField; private final String documentId; + private final String routingField; private final String routing; private final long bulkSize; private final boolean estimateBulkSizeUsingCompression; @@ -96,7 +98,6 @@ public class IndexConfiguration { private final boolean normalizeIndex; private static final String S3_PREFIX = "s3://"; - private static final String DEFAULT_AWS_REGION = "us-east-1"; @SuppressWarnings("unchecked") private IndexConfiguration(final Builder builder) { @@ -139,6 +140,7 @@ private IndexConfiguration(final Builder builder) { this.estimateBulkSizeUsingCompression = builder.estimateBulkSizeUsingCompression; this.maxLocalCompressionsForEstimation = builder.maxLocalCompressionsForEstimation; this.flushTimeout = builder.flushTimeout; + this.routingField = builder.routingField; this.routing = builder.routing; String documentIdField = builder.documentIdField; @@ -207,12 +209,21 @@ public static IndexConfiguration readIndexConfig(final OpenSearchSinkConfig open if (templateContent != null && templateFile != null) { LOG.warn("Both template_content and template_file are configured. Only template_content will be used"); } + + final String documentIdField = openSearchSinkConfig.getDocumentIdField(); final String documentId = openSearchSinkConfig.getDocumentId(); - if (documentId != null) { + if (documentIdField != null) { + LOG.warn("document_id_field is deprecated in favor of document_id, and support for document_id_field will be removed in a future major version release."); + builder = builder.withDocumentIdField(documentIdField); + } else if (documentId != null) { builder = builder.withDocumentId(documentId); } + final String routingField = openSearchSinkConfig.getRoutingField(); final String routing = openSearchSinkConfig.getRouting(); - if (routing != null) { + if (routingField != null) { + LOG.warn("routing_field is deprecated in favor of routing, and support for routing_field will be removed in a future major version release."); + builder = builder.withRoutingField(routingField); + } else if (routing != null) { builder = builder.withRouting(routing); } @@ -287,6 +298,10 @@ public String getDocumentIdField() { public String getDocumentId() { return documentId; } + public String getRoutingField() { + return routingField; + } + public String getRouting() { return routing; } @@ -416,6 +431,7 @@ public static class Builder { private String templateContent; private int numShards; private int numReplicas; + private String routingField; private String routing; private String pipeline; private String documentIdField; @@ -483,6 +499,11 @@ public Builder withDocumentId(final String documentId) { return this; } + public Builder withRoutingField(final String routingField) { + this.routingField = routingField; + return this; + } + public Builder withRouting(final String routing) { this.routing = routing; return this; From 6d7181fddb253ce1634436071e0db4809dcba28e Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Fri, 3 Jan 2025 12:09:36 -0800 Subject: [PATCH 43/62] fix integration tests to reuse deprecatd routing field and document id field Signed-off-by: Maxwell Brown --- .../sink/opensearch/OpenSearchSinkIT.java | 65 +++++++++++-------- .../opensearch/index/IndexConfiguration.java | 1 + 2 files changed, 40 insertions(+), 26 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index 8836031412..831bd6ef43 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -828,11 +828,10 @@ public void testOutputCustomIndex() throws IOException, InterruptedException { final String testTemplateFile = Objects.requireNonNull( getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE)).getFile(); final String testIdField = "someId"; - final String testDocumentID = "${/someId}"; final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); - metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); @@ -856,11 +855,10 @@ public void testOpenSearchBulkActionsCreate() throws IOException, InterruptedExc final String testTemplateFile = Objects.requireNonNull( getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE)).getFile(); final String testIdField = "someId"; - final String testDocumentID = "${/someId}"; final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); - metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); metadata.put(IndexConfiguration.ACTION, OpenSearchBulkActions.CREATE.toString()); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); @@ -885,11 +883,10 @@ public void testOpenSearchBulkActionsCreateWithExpression() throws IOException, final String testTemplateFile = Objects.requireNonNull( getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE)).getFile(); final String testIdField = "someId"; - final String testDocumentID = "${/someId}"; final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); - metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); Event event = (Event) testRecords.get(0).getData(); event.getMetadata().setAttribute("action", "create");metadata.put(IndexConfiguration.ACTION, "create"); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); @@ -915,11 +912,10 @@ public void testOpenSearchBulkActionsCreateWithInvalidExpression() throws IOExce final String testTemplateFile = Objects.requireNonNull( getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE)).getFile(); final String testIdField = "someId"; - final String testDocumentID = "${/someId}"; final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); - metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); metadata.put(IndexConfiguration.ACTION, "unknown"); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); assertThrows(IllegalArgumentException.class, () -> createObjectUnderTest(openSearchSinkConfig, true)); @@ -931,12 +927,11 @@ public void testBulkActionCreateWithActions() throws IOException, InterruptedExc final String testTemplateFile = Objects.requireNonNull( getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE)).getFile(); final String testIdField = "someId"; - final String testDocumentID = "${/someId}"; final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); - metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.CREATE.toString()); @@ -966,12 +961,11 @@ public void testBulkActionUpdateWithActions() throws IOException, InterruptedExc getClass().getClassLoader().getResource(TEST_TEMPLATE_BULK_FILE)).getFile(); final String testIdField = "someId"; - final String testDocumentID = "${/someId}"; final String testId = "foo"; List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson2(testIdField, testId, "name", "value1"))); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); - metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.CREATE.toString()); @@ -1017,7 +1011,6 @@ public void testBulkActionUpdateWithDocumentRootKey() throws IOException, Interr getClass().getClassLoader().getResource(TEST_TEMPLATE_BULK_FILE)).getFile(); final String testIdField = "someId"; - final String testDocumentID = "${/someId}"; final String testId = "foo"; final String documentRootKey = "root_key"; @@ -1032,7 +1025,7 @@ public void testBulkActionUpdateWithDocumentRootKey() throws IOException, Interr Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); metadata.put(IndexConfiguration.DOCUMENT_ROOT_KEY, documentRootKey); - metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); List> aList = new ArrayList<>(); Map actionMap = new HashMap<>(); actionMap.put("type", OpenSearchBulkActions.CREATE.toString()); @@ -1086,7 +1079,6 @@ public void testBulkActionUpsertWithActionsAndNoCreate() throws IOException, Int getClass().getClassLoader().getResource(TEST_TEMPLATE_BULK_FILE)).getFile(); final String testIdField = "someId"; - final String testDocumentID = "${/someId}"; final String testId = "foo"; List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson2(testIdField, testId, "key", "value"))); @@ -1096,7 +1088,7 @@ public void testBulkActionUpsertWithActionsAndNoCreate() throws IOException, Int aList.add(actionMap); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); - metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); metadata.put(IndexConfiguration.ACTIONS, aList); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); @@ -1119,12 +1111,11 @@ public void testBulkActionUpsertWithActions() throws IOException, InterruptedExc getClass().getClassLoader().getResource(TEST_TEMPLATE_BULK_FILE)).getFile(); final String testIdField = "someId"; - final String testDocumentID = "${/someId}"; final String testId = "foo"; List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson2(testIdField, testId, "name", "value1"))); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); - metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.CREATE.toString()); @@ -1172,11 +1163,10 @@ public void testBulkActionUpsertWithoutCreate() throws IOException, InterruptedE getClass().getClassLoader().getResource(TEST_TEMPLATE_BULK_FILE)).getFile(); final String testIdField = "someId"; - final String testDocumentID = "${/someId}"; final String testId = "foo"; List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson3(testIdField, testId, "name", "value1", "newKey", "newValue"))); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); - metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.UPSERT.toString()); @@ -1209,12 +1199,11 @@ public void testBulkActionDeleteWithActions() throws IOException, InterruptedExc getClass().getClassLoader().getResource(TEST_TEMPLATE_BULK_FILE)).getFile(); final String testIdField = "someId"; - final String testDocumentID = "${/someId}"; final String testId = "foo"; List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); - metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.DELETE.toString()); @@ -1321,12 +1310,11 @@ public void testOpenSearchDocumentId(final String testDocumentIdField) throws IO .withEventType("event") .build(); testEvent.put(testDocumentIdField, expectedId); - String testDocumentId = "${/" + testDocumentIdField + "}"; final List> testRecords = Collections.singletonList(new Record<>(testEvent)); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, null); - metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentId); + metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentIdField); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); @@ -1338,6 +1326,32 @@ public void testOpenSearchDocumentId(final String testDocumentIdField) throws IO sink.shutdown(); } + @ParameterizedTest + @ValueSource(strings = {"info/ids/rid", "rid"}) + public void testOpenSearchRoutingField(final String testRoutingField) throws IOException, InterruptedException { + final String expectedRoutingField = UUID.randomUUID().toString(); + final String testIndexAlias = "test_index"; + final Event testEvent = JacksonEvent.builder() + .withData(Map.of("arbitrary_data", UUID.randomUUID().toString())) + .withEventType("event") + .build(); + testEvent.put(testRoutingField, expectedRoutingField); + + final List> testRecords = Collections.singletonList(new Record<>(testEvent)); + + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, null); + metadata.put(IndexConfiguration.ROUTING_FIELD, testRoutingField); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); + sink.output(testRecords); + + final List routingFields = getSearchResponseRoutingFields(testIndexAlias); + for (String routingField : routingFields) { + assertThat(routingField, equalTo(expectedRoutingField)); + } + sink.shutdown(); + } + @ParameterizedTest @ValueSource(strings = {"", "info/ids/rid", "rid"}) public void testOpenSearchRouting(final String testRouting) throws IOException, InterruptedException { @@ -1612,7 +1626,6 @@ public void testOutputManagementDisabled() throws IOException, InterruptedExcept securityAccessor.createUser(username, password, roleName); final String testIdField = "someId"; - final String testDocumentID = "${/someId}"; final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); @@ -1621,7 +1634,7 @@ public void testOutputManagementDisabled() throws IOException, InterruptedExcept metadata.put(IndexConfiguration.INDEX_TYPE, IndexType.MANAGEMENT_DISABLED.getValue()); metadata.put(USERNAME, username); metadata.put(PASSWORD, password); - metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java index e946323439..7aaa779181 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java @@ -60,6 +60,7 @@ public class IndexConfiguration { public static final String FLUSH_TIMEOUT = "flush_timeout"; public static final String DOCUMENT_ID_FIELD = "document_id_field"; public static final String DOCUMENT_ID = "document_id"; + public static final String ROUTING_FIELD = "routing_field"; public static final String ROUTING = "routing"; public static final String PIPELINE = "pipeline"; public static final String ISM_POLICY_FILE = "ism_policy_file"; From e484b7d2477128bf511e153e5612aee93f39c98c Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Fri, 3 Jan 2025 12:23:22 -0800 Subject: [PATCH 44/62] action to enum integratin test fixes Signed-off-by: Maxwell Brown --- .../sink/opensearch/OpenSearchSinkIT.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index 831bd6ef43..df21bb115a 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -859,7 +859,7 @@ public void testOpenSearchBulkActionsCreate() throws IOException, InterruptedExc final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); - metadata.put(IndexConfiguration.ACTION, OpenSearchBulkActions.CREATE.toString()); + metadata.put(IndexConfiguration.ACTION, OpenSearchBulkActions.CREATE); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); @@ -934,7 +934,7 @@ public void testBulkActionCreateWithActions() throws IOException, InterruptedExc metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); - aMap.put("type", OpenSearchBulkActions.CREATE.toString()); + aMap.put("type", OpenSearchBulkActions.CREATE); aList.add(aMap); metadata.put(IndexConfiguration.ACTIONS, aList); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); @@ -968,7 +968,7 @@ public void testBulkActionUpdateWithActions() throws IOException, InterruptedExc metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); - aMap.put("type", OpenSearchBulkActions.CREATE.toString()); + aMap.put("type", OpenSearchBulkActions.CREATE); aList.add(aMap); metadata.put(IndexConfiguration.ACTIONS, aList); OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); @@ -989,7 +989,7 @@ public void testBulkActionUpdateWithActions() throws IOException, InterruptedExc testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson2(testIdField, testId, "name", "value2"))); aList = new ArrayList<>(); aMap = new HashMap<>(); - aMap.put("type", OpenSearchBulkActions.UPDATE.toString()); + aMap.put("type", OpenSearchBulkActions.UPDATE); aList.add(aMap); metadata.put(IndexConfiguration.ACTIONS, aList); openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); @@ -1028,7 +1028,7 @@ public void testBulkActionUpdateWithDocumentRootKey() throws IOException, Interr metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); List> aList = new ArrayList<>(); Map actionMap = new HashMap<>(); - actionMap.put("type", OpenSearchBulkActions.CREATE.toString()); + actionMap.put("type", OpenSearchBulkActions.CREATE); aList.add(actionMap); @@ -1056,7 +1056,7 @@ public void testBulkActionUpdateWithDocumentRootKey() throws IOException, Interr testRecords = Collections.singletonList(jsonStringToRecord(updateJsonEvent)); aList = new ArrayList<>(); actionMap = new HashMap<>(); - actionMap.put("type", OpenSearchBulkActions.UPDATE.toString()); + actionMap.put("type", OpenSearchBulkActions.UPDATE); aList.add(actionMap); metadata.put(IndexConfiguration.ACTIONS, aList); openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); @@ -1084,7 +1084,7 @@ public void testBulkActionUpsertWithActionsAndNoCreate() throws IOException, Int List> aList = new ArrayList<>(); Map actionMap = new HashMap<>(); - actionMap.put("type", OpenSearchBulkActions.UPSERT.toString()); + actionMap.put("type", OpenSearchBulkActions.UPSERT); aList.add(actionMap); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); @@ -1118,7 +1118,7 @@ public void testBulkActionUpsertWithActions() throws IOException, InterruptedExc metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); - aMap.put("type", OpenSearchBulkActions.CREATE.toString()); + aMap.put("type", OpenSearchBulkActions.CREATE); aList.add(aMap); metadata.put(IndexConfiguration.ACTIONS, aList); OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); @@ -1140,7 +1140,7 @@ public void testBulkActionUpsertWithActions() throws IOException, InterruptedExc testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson3(testIdField, testId, "name", "value3", "newKey", "newValue"))); aList = new ArrayList<>(); aMap = new HashMap<>(); - aMap.put("type", OpenSearchBulkActions.UPSERT.toString()); + aMap.put("type", OpenSearchBulkActions.UPSERT); aList.add(aMap); metadata.put(IndexConfiguration.ACTIONS, aList); openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); @@ -1169,7 +1169,7 @@ public void testBulkActionUpsertWithoutCreate() throws IOException, InterruptedE metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); - aMap.put("type", OpenSearchBulkActions.UPSERT.toString()); + aMap.put("type", OpenSearchBulkActions.UPSERT); aList.add(aMap); metadata.put(IndexConfiguration.ACTIONS, aList); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); @@ -1206,7 +1206,7 @@ public void testBulkActionDeleteWithActions() throws IOException, InterruptedExc metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); - aMap.put("type", OpenSearchBulkActions.DELETE.toString()); + aMap.put("type", OpenSearchBulkActions.DELETE); aList.add(aMap); metadata.put(IndexConfiguration.ACTIONS, aList); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); From c8636fde0d2b20850ae49d599816893f9e7c02f8 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Fri, 3 Jan 2025 12:57:37 -0800 Subject: [PATCH 45/62] string should be fine for integration test Signed-off-by: Maxwell Brown --- .../sink/opensearch/OpenSearchSinkIT.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index df21bb115a..831bd6ef43 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -859,7 +859,7 @@ public void testOpenSearchBulkActionsCreate() throws IOException, InterruptedExc final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); - metadata.put(IndexConfiguration.ACTION, OpenSearchBulkActions.CREATE); + metadata.put(IndexConfiguration.ACTION, OpenSearchBulkActions.CREATE.toString()); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); @@ -934,7 +934,7 @@ public void testBulkActionCreateWithActions() throws IOException, InterruptedExc metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); - aMap.put("type", OpenSearchBulkActions.CREATE); + aMap.put("type", OpenSearchBulkActions.CREATE.toString()); aList.add(aMap); metadata.put(IndexConfiguration.ACTIONS, aList); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); @@ -968,7 +968,7 @@ public void testBulkActionUpdateWithActions() throws IOException, InterruptedExc metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); - aMap.put("type", OpenSearchBulkActions.CREATE); + aMap.put("type", OpenSearchBulkActions.CREATE.toString()); aList.add(aMap); metadata.put(IndexConfiguration.ACTIONS, aList); OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); @@ -989,7 +989,7 @@ public void testBulkActionUpdateWithActions() throws IOException, InterruptedExc testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson2(testIdField, testId, "name", "value2"))); aList = new ArrayList<>(); aMap = new HashMap<>(); - aMap.put("type", OpenSearchBulkActions.UPDATE); + aMap.put("type", OpenSearchBulkActions.UPDATE.toString()); aList.add(aMap); metadata.put(IndexConfiguration.ACTIONS, aList); openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); @@ -1028,7 +1028,7 @@ public void testBulkActionUpdateWithDocumentRootKey() throws IOException, Interr metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); List> aList = new ArrayList<>(); Map actionMap = new HashMap<>(); - actionMap.put("type", OpenSearchBulkActions.CREATE); + actionMap.put("type", OpenSearchBulkActions.CREATE.toString()); aList.add(actionMap); @@ -1056,7 +1056,7 @@ public void testBulkActionUpdateWithDocumentRootKey() throws IOException, Interr testRecords = Collections.singletonList(jsonStringToRecord(updateJsonEvent)); aList = new ArrayList<>(); actionMap = new HashMap<>(); - actionMap.put("type", OpenSearchBulkActions.UPDATE); + actionMap.put("type", OpenSearchBulkActions.UPDATE.toString()); aList.add(actionMap); metadata.put(IndexConfiguration.ACTIONS, aList); openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); @@ -1084,7 +1084,7 @@ public void testBulkActionUpsertWithActionsAndNoCreate() throws IOException, Int List> aList = new ArrayList<>(); Map actionMap = new HashMap<>(); - actionMap.put("type", OpenSearchBulkActions.UPSERT); + actionMap.put("type", OpenSearchBulkActions.UPSERT.toString()); aList.add(actionMap); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); @@ -1118,7 +1118,7 @@ public void testBulkActionUpsertWithActions() throws IOException, InterruptedExc metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); - aMap.put("type", OpenSearchBulkActions.CREATE); + aMap.put("type", OpenSearchBulkActions.CREATE.toString()); aList.add(aMap); metadata.put(IndexConfiguration.ACTIONS, aList); OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); @@ -1140,7 +1140,7 @@ public void testBulkActionUpsertWithActions() throws IOException, InterruptedExc testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson3(testIdField, testId, "name", "value3", "newKey", "newValue"))); aList = new ArrayList<>(); aMap = new HashMap<>(); - aMap.put("type", OpenSearchBulkActions.UPSERT); + aMap.put("type", OpenSearchBulkActions.UPSERT.toString()); aList.add(aMap); metadata.put(IndexConfiguration.ACTIONS, aList); openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); @@ -1169,7 +1169,7 @@ public void testBulkActionUpsertWithoutCreate() throws IOException, InterruptedE metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); - aMap.put("type", OpenSearchBulkActions.UPSERT); + aMap.put("type", OpenSearchBulkActions.UPSERT.toString()); aList.add(aMap); metadata.put(IndexConfiguration.ACTIONS, aList); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); @@ -1206,7 +1206,7 @@ public void testBulkActionDeleteWithActions() throws IOException, InterruptedExc metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); - aMap.put("type", OpenSearchBulkActions.DELETE); + aMap.put("type", OpenSearchBulkActions.DELETE.toString()); aList.add(aMap); metadata.put(IndexConfiguration.ACTIONS, aList); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); From da251703f2f355619c07ab9d8222c1a75b482afc Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Fri, 3 Jan 2025 13:38:58 -0800 Subject: [PATCH 46/62] enum string check Signed-off-by: Maxwell Brown --- .../opensearch/configuration/ActionConfiguration.java | 8 ++++++++ .../opensearch/configuration/OpenSearchSinkConfig.java | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ActionConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ActionConfiguration.java index 96519fceaa..0f9db9984e 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ActionConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ActionConfiguration.java @@ -12,6 +12,14 @@ public class ActionConfiguration { @JsonProperty("type") private OpenSearchBulkActions type; + @AssertTrue(message = "type must be one of index, create, update, upsert, delete") + boolean isTypeValid() { + if (type == null) { //type will be null if the string doesnt match one of the enums + return true; + } + return false; + } + public String getType() { return type.toString(); } diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java index a14fb38cd7..ee424da181 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java @@ -139,6 +139,14 @@ public boolean getEnableRequestCompression() { @JsonProperty("action") private OpenSearchBulkActions action = OpenSearchBulkActions.INDEX; + @AssertTrue(message = "action must be one of index, create, update, upsert, delete") + boolean isActionValid() { + if (action == null) { //action will be null if the string doesn't match one of the enums + return true; + } + return false; + } + public String getAction() { return action.toString(); } From df7c9295627aad52e9b2829d16cf2eb0ba0ba6e4 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Fri, 3 Jan 2025 13:54:18 -0800 Subject: [PATCH 47/62] fix enum string check Signed-off-by: Maxwell Brown --- .../opensearch/configuration/ActionConfiguration.java | 11 +++++------ .../configuration/OpenSearchSinkConfig.java | 4 ++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ActionConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ActionConfiguration.java index 0f9db9984e..f867db8c50 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ActionConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ActionConfiguration.java @@ -4,7 +4,6 @@ import jakarta.validation.constraints.AssertTrue; import jakarta.validation.constraints.Size; import lombok.Getter; -import org.apache.commons.lang3.EnumUtils; import org.opensearch.dataprepper.model.opensearch.OpenSearchBulkActions; public class ActionConfiguration { @@ -12,12 +11,12 @@ public class ActionConfiguration { @JsonProperty("type") private OpenSearchBulkActions type; - @AssertTrue(message = "type must be one of index, create, update, upsert, delete") - boolean isTypeValid() { - if (type == null) { //type will be null if the string doesnt match one of the enums - return true; + @AssertTrue(message = "action must be one of index, create, update, upsert, delete") + boolean isActionValid() { + if (type == null) { //type will be null if the string doesn't match one of the enums + return false; } - return false; + return true; } public String getType() { diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java index ee424da181..d988cc3a6e 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java @@ -142,9 +142,9 @@ public boolean getEnableRequestCompression() { @AssertTrue(message = "action must be one of index, create, update, upsert, delete") boolean isActionValid() { if (action == null) { //action will be null if the string doesn't match one of the enums - return true; + return false; } - return false; + return true; } public String getAction() { From 018857c12989d48d29a21caff59a0e9d35c44ece Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Fri, 3 Jan 2025 13:57:28 -0800 Subject: [PATCH 48/62] document id field should have testIdField as value in Integration Tests Signed-off-by: Maxwell Brown --- .../sink/opensearch/OpenSearchSinkIT.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index 831bd6ef43..80b15162d5 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -831,7 +831,7 @@ public void testOutputCustomIndex() throws IOException, InterruptedException { final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); - metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); @@ -858,7 +858,7 @@ public void testOpenSearchBulkActionsCreate() throws IOException, InterruptedExc final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); - metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); metadata.put(IndexConfiguration.ACTION, OpenSearchBulkActions.CREATE.toString()); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); @@ -886,7 +886,7 @@ public void testOpenSearchBulkActionsCreateWithExpression() throws IOException, final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); - metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); Event event = (Event) testRecords.get(0).getData(); event.getMetadata().setAttribute("action", "create");metadata.put(IndexConfiguration.ACTION, "create"); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); @@ -915,7 +915,7 @@ public void testOpenSearchBulkActionsCreateWithInvalidExpression() throws IOExce final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); - metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); metadata.put(IndexConfiguration.ACTION, "unknown"); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); assertThrows(IllegalArgumentException.class, () -> createObjectUnderTest(openSearchSinkConfig, true)); @@ -931,7 +931,7 @@ public void testBulkActionCreateWithActions() throws IOException, InterruptedExc final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); - metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.CREATE.toString()); @@ -965,7 +965,7 @@ public void testBulkActionUpdateWithActions() throws IOException, InterruptedExc List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson2(testIdField, testId, "name", "value1"))); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); - metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.CREATE.toString()); @@ -1025,7 +1025,7 @@ public void testBulkActionUpdateWithDocumentRootKey() throws IOException, Interr Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); metadata.put(IndexConfiguration.DOCUMENT_ROOT_KEY, documentRootKey); - metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); List> aList = new ArrayList<>(); Map actionMap = new HashMap<>(); actionMap.put("type", OpenSearchBulkActions.CREATE.toString()); @@ -1088,7 +1088,7 @@ public void testBulkActionUpsertWithActionsAndNoCreate() throws IOException, Int aList.add(actionMap); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); - metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); metadata.put(IndexConfiguration.ACTIONS, aList); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); @@ -1115,7 +1115,7 @@ public void testBulkActionUpsertWithActions() throws IOException, InterruptedExc List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson2(testIdField, testId, "name", "value1"))); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); - metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.CREATE.toString()); @@ -1166,7 +1166,7 @@ public void testBulkActionUpsertWithoutCreate() throws IOException, InterruptedE final String testId = "foo"; List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson3(testIdField, testId, "name", "value1", "newKey", "newValue"))); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); - metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.UPSERT.toString()); @@ -1203,7 +1203,7 @@ public void testBulkActionDeleteWithActions() throws IOException, InterruptedExc List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); - metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.DELETE.toString()); @@ -1634,7 +1634,7 @@ public void testOutputManagementDisabled() throws IOException, InterruptedExcept metadata.put(IndexConfiguration.INDEX_TYPE, IndexType.MANAGEMENT_DISABLED.getValue()); metadata.put(USERNAME, username); metadata.put(PASSWORD, password); - metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testId); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); From b1202b0c47b7ba66af2c3d4e34c9fbeefdf742a8 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Fri, 3 Jan 2025 14:04:48 -0800 Subject: [PATCH 49/62] integraton tests should pass now Signed-off-by: Maxwell Brown --- .../plugins/sink/opensearch/OpenSearchSinkIT.java | 4 ++-- .../sink/opensearch/OpenSearchSinkConfigurationTests.java | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index 80b15162d5..2e83534ba0 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -918,7 +918,7 @@ public void testOpenSearchBulkActionsCreateWithInvalidExpression() throws IOExce metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); metadata.put(IndexConfiguration.ACTION, "unknown"); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); - assertThrows(IllegalArgumentException.class, () -> createObjectUnderTest(openSearchSinkConfig, true)); + assertThrows(NullPointerException.class, () -> createObjectUnderTest(openSearchSinkConfig, true)); } @Test @@ -1314,7 +1314,7 @@ public void testOpenSearchDocumentId(final String testDocumentIdField) throws IO final List> testRecords = Collections.singletonList(new Record<>(testEvent)); Map metadata = initializeConfigurationMetadata(null, testIndexAlias, null); - metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentIdField); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testDocumentIdField); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfigurationTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfigurationTests.java index cfe97dbbbe..22e1df982e 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfigurationTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfigurationTests.java @@ -45,25 +45,25 @@ public void testReadESConfig() throws IOException { assertEquals(OpenSearchBulkActions.INDEX.toString(), openSearchSinkConfiguration.getIndexConfiguration().getAction()); } - @Test(expected = IllegalArgumentException.class) + @Test(expected = NullPointerException.class) public void testInvalidAction() throws IOException { OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSinkConfig(INVALID_ACTION_CONFIG)); } - @Test(expected = IllegalArgumentException.class) + @Test(expected = NullPointerException.class) public void testInvalidActions() throws IOException { OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSinkConfig(INVALID_ACTIONS_CONFIG)); } - @Test(expected = IllegalArgumentException.class) + @Test(expected = NullPointerException.class) public void testInvalidActionWithExpression() throws IOException { expressionEvaluator = mock(ExpressionEvaluator.class); when(expressionEvaluator.isValidExpressionStatement(anyString())).thenReturn(false); OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSinkConfig(INVALID_ACTION_WITH_EXPRESSION_CONFIG), expressionEvaluator); } - @Test(expected = IllegalArgumentException.class) + @Test(expected = NullPointerException.class) public void testInvalidActionsWithExpression() throws IOException { expressionEvaluator = mock(ExpressionEvaluator.class); when(expressionEvaluator.isValidExpressionStatement(anyString())).thenReturn(false); From 5d59aefa8942764bd9a44cbf04bda92049301468 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Fri, 3 Jan 2025 14:48:10 -0800 Subject: [PATCH 50/62] Connection configuration switched back to using metadata Signed-off-by: Maxwell Brown --- .../AwsAuthenticationConfiguration.java | 4 +- .../configuration/OpenSearchSinkConfig.java | 2 +- .../ConnectionConfigurationTests.java | 364 ++++++++++++------ 3 files changed, 257 insertions(+), 113 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AwsAuthenticationConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AwsAuthenticationConfiguration.java index d99e494179..5e8bcd8b63 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AwsAuthenticationConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AwsAuthenticationConfiguration.java @@ -11,11 +11,13 @@ import java.util.Map; +import static org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig.DEFAULT_AWS_REGION; + public class AwsAuthenticationConfiguration { @JsonProperty("region") @Size(min = 1, message = "Region cannot be empty string") - private String awsRegion; + private String awsRegion = DEFAULT_AWS_REGION; @JsonProperty("sts_role_arn") @Size(min = 20, max = 2048, message = "awsStsRoleArn length should be between 1 and 2048 characters") diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java index d988cc3a6e..1b21f63f68 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java @@ -18,7 +18,7 @@ public class OpenSearchSinkConfig { public static final boolean DEFAULT_ESTIMATE_BULK_SIZE_USING_COMPRESSION = false; public static final int DEFAULT_MAX_LOCAL_COMPRESSIONS_FOR_ESTIMATION = 2; public static final long DEFAULT_FLUSH_TIMEOUT = 60_000L; - private static final String DEFAULT_AWS_REGION = "us-east-1"; + public static final String DEFAULT_AWS_REGION = "us-east-1"; @Getter @JsonProperty("hosts") private List hosts; diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java index ffb34f9397..dcc1d2b655 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java @@ -5,8 +5,8 @@ package org.opensearch.dataprepper.plugins.sink.opensearch; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; @@ -25,12 +25,14 @@ import software.amazon.awssdk.http.apache.ApacheHttpClient; import software.amazon.awssdk.regions.Region; -import java.io.File; import java.io.IOException; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; +import java.util.UUID; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; @@ -49,45 +51,19 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.sink.opensearch.ConnectionConfiguration.SERVERLESS; +import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.DISTRIBUTION_VERSION; @ExtendWith(MockitoExtension.class) class ConnectionConfigurationTests { - private static final String OPEN_SEARCH_SINK_CONFIGURATIONS = "open-search-sink-configurations.yaml"; - private static final String EMPTY_SINK_CONFIG = "empty-sink"; - private static final String ES6_DEFAULT_CONFIG = "es6-default"; - private static final String AWS_SERVERLESS_DEFAULT = "aws-serverless-default"; - private static final String AWS_SERVERLESS_NO_CERT = "aws-serverless-no-cert"; - private static final String BASIC_CREDENTIALS_NO_CERT = "basic-credentials-no-cert"; - private static final String BASIC_CREDENTIALS_NO_CERT_INSECURE = "basic-credentials-no-cert-insecure"; - private static final String BASIC_CREDENTIALS_WITH_CERT = "basic-credentials-with-cert"; - private static final String AWS_REGION_ONLY = "aws-region-only"; - private static final String SERVERLESS_OPTIONS = "serverless-options"; - private static final String AWS_REGION_ONLY_INSECURE = "aws-region-only-insecure"; - private static final String AWS_WITH_CERT = "aws-with-cert"; - private static final String AWS_WITH_CERT_AND_ARN = "aws-with-cert-and-arn"; - private static final String AWS_WITH_2_HEADER = "aws-with-2-header"; - private static final String VALID_PROXY_IP = "valid-proxy-ip"; - private static final String VALID_PROXY_HOST_NAME = "valid-proxy-host-name"; - private static final String VALID_HTTP_PROXY_SCHEME = "valid-http-proxy-scheme"; - private static final String INVALID_HTTP_PROXY_PORT = "invalid-http-proxy-port"; - private static final String INVALID_HTTP_PROXY_NO_PORT = "invalid-http-proxy-no-port"; - private static final String INVALID_HTTP_PROXY_PORT_NOT_IN_RANGE = "invalid-http-proxy-port-not-in-range"; - private static final String INVALID_HTTP_PROXY_NOT_HTTP = "invalid-http-proxy-not-http"; - + private static final String PROXY_PARAMETER = "proxy"; private final List TEST_HOSTS = Collections.singletonList("http://localhost:9200"); - private final String TEST_USERNAME = "test-username"; - private final String TEST_PASSWORD = "test-password"; + private final String TEST_USERNAME = "admin"; + private final String TEST_PASSWORD = "admin"; private final Integer TEST_CONNECT_TIMEOUT = 5; private final Integer TEST_SOCKET_TIMEOUT = 10; + private final String TEST_CERT_PATH = Objects.requireNonNull(getClass().getClassLoader().getResource("test-ca.pem")).getFile(); private final String TEST_ROLE = "arn:aws:iam::123456789012:role/test-role"; - private final String TEST_NETWORK_POLICY = "test network policy"; - private final String TEST_COLLECTION_NAME = "test collection"; - private final String TEST_VPCE_ID = "test vpce id"; - private final String TEST_EXTERNAL_ID = "test-external-id"; - private final String TEST_HEADER_NAME_1 = "header1"; - private final String TEST_HEADER_NAME_2 = "header2"; - private final String TEST_HEADER_VALUE_1 = "test-header-1"; - private final String TEST_HEADER_VALUE_2 = "test-header-2"; @Mock private ApacheHttpClient.Builder apacheHttpClientBuilder; @@ -95,12 +71,13 @@ class ConnectionConfigurationTests { private ApacheHttpClient apacheHttpClient; @Mock private AwsCredentialsSupplier awsCredentialsSupplier; - + ObjectMapper objectMapper; @Test - void testReadConnectionConfigurationDefault() throws IOException { - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(EMPTY_SINK_CONFIG); + void testReadConnectionConfigurationDefault() throws JsonProcessingException { + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( + TEST_HOSTS, null, null, null, null, false, null, null, null, false); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertEquals(TEST_HOSTS, connectionConfiguration.getHosts()); @@ -114,16 +91,21 @@ void testReadConnectionConfigurationDefault() throws IOException { } @Test - void testReadConnectionConfigurationES6Default() throws IOException { - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(ES6_DEFAULT_CONFIG); + void testReadConnectionConfigurationES6Default() throws JsonProcessingException { + final Map configMetadata = generateConfigurationMetadata( + TEST_HOSTS, null, null, null, null, true, null, null, null, false); + configMetadata.put(DISTRIBUTION_VERSION, "es6"); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(configMetadata); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertFalse(connectionConfiguration.isRequestCompressionEnabled()); } @Test - void testReadConnectionConfigurationAwsOptionServerlessDefault() throws IOException { - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(AWS_SERVERLESS_DEFAULT); + void testReadConnectionConfigurationAwsOptionServerlessDefault() throws JsonProcessingException { + final String testArn = TEST_ROLE; + final Map configMetadata = generateConfigurationMetadataWithAwsOption(TEST_HOSTS, null, null, null, null, true, false, null, testArn, null, TEST_CERT_PATH, false, Collections.emptyMap()); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(configMetadata); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertTrue(connectionConfiguration.isServerless()); @@ -131,7 +113,8 @@ void testReadConnectionConfigurationAwsOptionServerlessDefault() throws IOExcept @Test void testCreateClientDefault() throws IOException { - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(EMPTY_SINK_CONFIG); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( + TEST_HOSTS, null, null, null, null, false, null, null, null, false); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); @@ -141,7 +124,8 @@ void testCreateClientDefault() throws IOException { @Test void testCreateOpenSearchClientDefault() throws IOException { - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(EMPTY_SINK_CONFIG); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( + TEST_HOSTS, null, null, null, null, false, null, null, null, false); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); @@ -155,9 +139,13 @@ void testCreateOpenSearchClientDefault() throws IOException { @Test void testCreateOpenSearchClientAwsServerlessDefault() throws IOException { - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(AWS_SERVERLESS_NO_CERT); + final Map configMetadata = generateConfigurationMetadata( + TEST_HOSTS, null, null, null, null, true, null, null, null, false); + configMetadata.put(SERVERLESS, true); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(configMetadata); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); + final AwsCredentialsProvider awsCredentialsProvider = mock(AwsCredentialsProvider.class); when(awsCredentialsSupplier.getProvider(any())).thenReturn(awsCredentialsProvider); @@ -179,8 +167,9 @@ void testCreateOpenSearchClientAwsServerlessDefault() throws IOException { } @Test - void testReadConnectionConfigurationWithBasicCredentialsAndNoCert() throws IOException { - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(BASIC_CREDENTIALS_NO_CERT); + void testReadConnectionConfigurationWithDeprecatedBasicCredentialsAndNoCert() throws JsonProcessingException { + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( + TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, null, false); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertEquals(TEST_HOSTS, connectionConfiguration.getHosts()); @@ -191,10 +180,24 @@ void testReadConnectionConfigurationWithBasicCredentialsAndNoCert() throws IOExc assertFalse(connectionConfiguration.isAwsSigv4()); } + @Test + void testCreateClientWithDeprecatedBasicCredentialsAndNoCert() throws IOException { + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( + TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, null, false); + final ConnectionConfiguration connectionConfiguration = + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); + final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); + assertNotNull(client); + client.close(); + } @Test void testCreateClientWithBasicCredentialsAndNoCert() throws IOException { - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(BASIC_CREDENTIALS_NO_CERT); + final Map configurationMetadata = generateConfigurationMetadata( + TEST_HOSTS, null, null, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, null, false); + configurationMetadata.put("username", TEST_USERNAME); + configurationMetadata.put("password", TEST_PASSWORD); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(configurationMetadata); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); @@ -202,10 +205,10 @@ void testCreateClientWithBasicCredentialsAndNoCert() throws IOException { client.close(); } - @Test void testCreateOpenSearchClientNoCert() throws IOException { - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(BASIC_CREDENTIALS_NO_CERT); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( + TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, null, false); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); @@ -218,7 +221,8 @@ void testCreateOpenSearchClientNoCert() throws IOException { @Test void testCreateClientInsecure() throws IOException { - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(BASIC_CREDENTIALS_NO_CERT_INSECURE); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( + TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, null, true); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); @@ -228,7 +232,8 @@ void testCreateClientInsecure() throws IOException { @Test void testCreateOpenSearchClientInsecure() throws IOException { - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(BASIC_CREDENTIALS_NO_CERT_INSECURE); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( + TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, null, true); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); @@ -241,7 +246,8 @@ void testCreateOpenSearchClientInsecure() throws IOException { @Test void testCreateClientWithCertPath() throws IOException { - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(BASIC_CREDENTIALS_WITH_CERT); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( + TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, false); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); @@ -251,7 +257,8 @@ void testCreateClientWithCertPath() throws IOException { @Test void testCreateOpenSearchClientWithCertPath() throws IOException { - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(BASIC_CREDENTIALS_WITH_CERT); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( + TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, false); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); @@ -264,26 +271,58 @@ void testCreateOpenSearchClientWithCertPath() throws IOException { @Test void testCreateClientWithAWSSigV4AndRegion() throws IOException { - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(AWS_REGION_ONLY); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( + TEST_HOSTS, null, null, null, null, true, "us-west-2", null, null, false); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); - assertEquals("us-east-2", connectionConfiguration.getAwsRegion()); + assertEquals("us-west-2", connectionConfiguration.getAwsRegion()); assertTrue(connectionConfiguration.isAwsSigv4()); } @Test void testServerlessOptions() throws IOException { - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(SERVERLESS_OPTIONS); + final String serverlessNetworkPolicyName = UUID.randomUUID().toString(); + final String serverlessCollectionName = UUID.randomUUID().toString(); + final String serverlessVpceId = UUID.randomUUID().toString(); + + final Map metadata = new HashMap<>(); + final Map awsOptionMetadata = new HashMap<>(); + final Map serverlessOptionsMetadata = new HashMap<>(); + serverlessOptionsMetadata.put("network_policy_name", serverlessNetworkPolicyName); + serverlessOptionsMetadata.put("collection_name", serverlessCollectionName); + serverlessOptionsMetadata.put("vpce_id", serverlessVpceId); + awsOptionMetadata.put("region", UUID.randomUUID().toString()); + awsOptionMetadata.put("serverless", true); + awsOptionMetadata.put("serverless_options", serverlessOptionsMetadata); + awsOptionMetadata.put("sts_role_arn", TEST_ROLE); + metadata.put("hosts", TEST_HOSTS); + metadata.put("username", UUID.randomUUID().toString()); + metadata.put("password", UUID.randomUUID().toString()); + metadata.put("connect_timeout", 1); + metadata.put("socket_timeout", 1); + metadata.put("aws", awsOptionMetadata); + + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(metadata); + final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); + assertThat(connectionConfiguration.getServerlessNetworkPolicyName(), equalTo(serverlessNetworkPolicyName)); + assertThat(connectionConfiguration.getServerlessCollectionName(), equalTo(serverlessCollectionName)); + assertThat(connectionConfiguration.getServerlessVpceId(), equalTo(serverlessVpceId)); + } + + @Test + void testCreateClientWithAWSSigV4DefaultRegion() throws IOException { + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( + TEST_HOSTS, null, null, null, null, true, null, null, null, false); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); - assertThat(connectionConfiguration.getServerlessNetworkPolicyName(), equalTo(TEST_NETWORK_POLICY)); - assertThat(connectionConfiguration.getServerlessCollectionName(), equalTo(TEST_COLLECTION_NAME)); - assertThat(connectionConfiguration.getServerlessVpceId(), equalTo(TEST_VPCE_ID)); + assertEquals("us-east-1", connectionConfiguration.getAwsRegion()); + assertTrue(connectionConfiguration.isAwsSigv4()); } @Test void testCreateClientWithAWSSigV4AndInsecure() throws IOException { - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(AWS_REGION_ONLY_INSECURE); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( + TEST_HOSTS, null, null, null, null, true, null, null, null, true); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertEquals("us-east-1", connectionConfiguration.getAwsRegion()); @@ -292,7 +331,8 @@ void testCreateClientWithAWSSigV4AndInsecure() throws IOException { @Test void testCreateClientWithAWSSigV4AndCertPath() throws IOException { - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(AWS_WITH_CERT); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( + TEST_HOSTS, null, null, null, null, true, null, null, TEST_CERT_PATH, false); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertEquals("us-east-1", connectionConfiguration.getAwsRegion()); @@ -300,8 +340,9 @@ void testCreateClientWithAWSSigV4AndCertPath() throws IOException { } @Test - void testCreateClientWithAWSSigV4AndSTSRole() throws IOException { - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(AWS_WITH_CERT_AND_ARN); + void testCreateClientWithAWSSigV4AndSTSRole() throws JsonProcessingException { + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( + TEST_HOSTS, null, null, null, null, true, null, TEST_ROLE, TEST_CERT_PATH, false); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertThat(connectionConfiguration, notNullValue()); @@ -324,10 +365,15 @@ void testCreateClientWithAWSSigV4AndSTSRole() throws IOException { } @Test - void testCreateOpenSearchClientWithAWSSigV4AndSTSRole() throws IOException { - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(AWS_WITH_CERT_AND_ARN); + void testCreateOpenSearchClientWithAWSSigV4AndSTSRole() throws JsonProcessingException { + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( + TEST_HOSTS, null, null, null, null, true, null, TEST_ROLE, TEST_CERT_PATH, false); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); + assertThat(connectionConfiguration, notNullValue()); + assertThat(connectionConfiguration.getAwsRegion(), equalTo("us-east-1")); + assertThat(connectionConfiguration.isAwsSigv4(), equalTo(true)); + assertThat(connectionConfiguration.getAwsStsRoleArn(), equalTo(TEST_ROLE)); final AwsCredentialsProvider awsCredentialsProvider = mock(AwsCredentialsProvider.class); when(awsCredentialsSupplier.getProvider(any())).thenReturn(awsCredentialsProvider); @@ -351,8 +397,15 @@ void testCreateOpenSearchClientWithAWSSigV4AndSTSRole() throws IOException { } @Test - void testCreateClientWithAWSOption() throws IOException { - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(AWS_WITH_2_HEADER); + void testCreateClientWithAWSOption() throws JsonProcessingException { + final String headerName1 = UUID.randomUUID().toString(); + final String headerValue1 = UUID.randomUUID().toString(); + final String headerName2 = UUID.randomUUID().toString(); + final String headerValue2 = UUID.randomUUID().toString(); + final String testArn = TEST_ROLE; + final String externalId = UUID.randomUUID().toString(); + final Map configurationMetadata = generateConfigurationMetadataWithAwsOption(TEST_HOSTS, null, null, null, null, false, true, null, testArn, externalId, null,false, Map.of(headerName1, headerValue1, headerName2, headerValue2)); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(configurationMetadata); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); @@ -370,18 +423,25 @@ void testCreateClientWithAWSOption() throws IOException { final AwsCredentialsOptions actualOptions = awsCredentialsOptionsArgumentCaptor.getValue(); assertThat(actualOptions.getStsRoleArn(), equalTo(TEST_ROLE)); - assertThat(actualOptions.getStsExternalId(), equalTo(TEST_EXTERNAL_ID)); + assertThat(actualOptions.getStsExternalId(), equalTo(externalId)); assertThat(actualOptions.getStsHeaderOverrides(), notNullValue()); assertThat(actualOptions.getStsHeaderOverrides().size(), equalTo(2)); - assertThat(actualOptions.getStsHeaderOverrides(), hasKey(TEST_HEADER_NAME_2)); - assertThat(actualOptions.getStsHeaderOverrides().get(TEST_HEADER_NAME_1), equalTo(TEST_HEADER_VALUE_1)); - assertThat(actualOptions.getStsHeaderOverrides(), hasKey(TEST_HEADER_NAME_2)); - assertThat(actualOptions.getStsHeaderOverrides().get(TEST_HEADER_NAME_2), equalTo(TEST_HEADER_VALUE_2)); + assertThat(actualOptions.getStsHeaderOverrides(), hasKey(headerName1)); + assertThat(actualOptions.getStsHeaderOverrides().get(headerName1), equalTo(headerValue1)); + assertThat(actualOptions.getStsHeaderOverrides(), hasKey(headerName2)); + assertThat(actualOptions.getStsHeaderOverrides().get(headerName2), equalTo(headerValue2)); } @Test - void testCreateOpenSearchClientWithAWSOption() throws IOException { - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(AWS_WITH_2_HEADER); + void testCreateOpenSearchClientWithAWSOption() throws JsonProcessingException { + final String headerName1 = UUID.randomUUID().toString(); + final String headerValue1 = UUID.randomUUID().toString(); + final String headerName2 = UUID.randomUUID().toString(); + final String headerValue2 = UUID.randomUUID().toString(); + final String testArn = TEST_ROLE; + final String externalId = UUID.randomUUID().toString(); + final Map configurationMetadata = generateConfigurationMetadataWithAwsOption(TEST_HOSTS, null, null, null, null, false, true, null, testArn, externalId, TEST_CERT_PATH, false, Map.of(headerName1, headerValue1, headerName2, headerValue2)); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(configurationMetadata); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); @@ -396,7 +456,7 @@ void testCreateOpenSearchClientWithAWSOption() throws IOException { final OpenSearchClient openSearchClient = connectionConfiguration.createOpenSearchClient(client, awsCredentialsSupplier); assertNotNull(openSearchClient); - assertThat(openSearchClient._transport(), instanceOf(AwsSdk2Transport.class)); + assertThat(openSearchClient._transport(), instanceOf(AwsSdk2Transport.class)); final AwsSdk2Transport opensearchTransport = (AwsSdk2Transport) openSearchClient._transport(); assertThat(opensearchTransport.options().credentials(), equalTo(awsCredentialsProvider)); @@ -407,15 +467,21 @@ void testCreateOpenSearchClientWithAWSOption() throws IOException { assertThat(actualOptions.getStsRoleArn(), equalTo(TEST_ROLE)); assertThat(actualOptions.getStsHeaderOverrides(), notNullValue()); assertThat(actualOptions.getStsHeaderOverrides().size(), equalTo(2)); - assertThat(actualOptions.getStsHeaderOverrides(), hasKey(TEST_HEADER_NAME_2)); - assertThat(actualOptions.getStsHeaderOverrides().get(TEST_HEADER_NAME_1), equalTo(TEST_HEADER_VALUE_1)); - assertThat(actualOptions.getStsHeaderOverrides(), hasKey(TEST_HEADER_NAME_2)); - assertThat(actualOptions.getStsHeaderOverrides().get(TEST_HEADER_NAME_2), equalTo(TEST_HEADER_VALUE_2)); + assertThat(actualOptions.getStsHeaderOverrides(), hasKey(headerName1)); + assertThat(actualOptions.getStsHeaderOverrides().get(headerName1), equalTo(headerValue1)); + assertThat(actualOptions.getStsHeaderOverrides(), hasKey(headerName2)); + assertThat(actualOptions.getStsHeaderOverrides().get(headerName2), equalTo(headerValue2)); } @Test - void testCreateClientWithAWSSigV4AndHeaderOverrides() throws IOException { - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(AWS_WITH_2_HEADER); + void testCreateClientWithAWSSigV4AndHeaderOverrides() throws JsonProcessingException { + final String headerName1 = UUID.randomUUID().toString(); + final String headerValue1 = UUID.randomUUID().toString(); + final String headerName2 = UUID.randomUUID().toString(); + final String headerValue2 = UUID.randomUUID().toString(); + final Map configurationMetadata = generateConfigurationMetadata(TEST_HOSTS, null, null, null, null, true, null, TEST_ROLE, TEST_CERT_PATH, false); + configurationMetadata.put("aws_sts_header_overrides", Map.of(headerName1, headerValue1, headerName2, headerValue2)); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(configurationMetadata); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); @@ -436,15 +502,21 @@ void testCreateClientWithAWSSigV4AndHeaderOverrides() throws IOException { assertThat(actualOptions.getRegion(), equalTo(Region.US_EAST_1)); assertThat(actualOptions.getStsHeaderOverrides(), notNullValue()); assertThat(actualOptions.getStsHeaderOverrides().size(), equalTo(2)); - assertThat(actualOptions.getStsHeaderOverrides(), hasKey(TEST_HEADER_NAME_2)); - assertThat(actualOptions.getStsHeaderOverrides().get(TEST_HEADER_NAME_1), equalTo(TEST_HEADER_VALUE_1)); - assertThat(actualOptions.getStsHeaderOverrides(), hasKey(TEST_HEADER_NAME_2)); - assertThat(actualOptions.getStsHeaderOverrides().get(TEST_HEADER_NAME_2), equalTo(TEST_HEADER_VALUE_2)); + assertThat(actualOptions.getStsHeaderOverrides(), hasKey(headerName1)); + assertThat(actualOptions.getStsHeaderOverrides().get(headerName1), equalTo(headerValue1)); + assertThat(actualOptions.getStsHeaderOverrides(), hasKey(headerName2)); + assertThat(actualOptions.getStsHeaderOverrides().get(headerName2), equalTo(headerValue2)); } @Test - void testCreateOpenSearchClientWithAWSSigV4AndHeaderOverrides() throws IOException { - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(AWS_WITH_2_HEADER); + void testCreateOpenSearchClientWithAWSSigV4AndHeaderOverrides() throws JsonProcessingException { + final String headerName1 = UUID.randomUUID().toString(); + final String headerValue1 = UUID.randomUUID().toString(); + final String headerName2 = UUID.randomUUID().toString(); + final String headerValue2 = UUID.randomUUID().toString(); + final Map configurationMetadata = generateConfigurationMetadata(TEST_HOSTS, null, null, null, null, true, null, TEST_ROLE, TEST_CERT_PATH, false); + configurationMetadata.put("aws_sts_header_overrides", Map.of(headerName1, headerValue1, headerName2, headerValue2)); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(configurationMetadata); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); @@ -472,16 +544,19 @@ void testCreateOpenSearchClientWithAWSSigV4AndHeaderOverrides() throws IOExcepti assertThat(actualOptions.getRegion(), equalTo(Region.US_EAST_1)); assertThat(actualOptions.getStsHeaderOverrides(), notNullValue()); assertThat(actualOptions.getStsHeaderOverrides().size(), equalTo(2)); - assertThat(actualOptions.getStsHeaderOverrides(), hasKey(TEST_HEADER_NAME_2)); - assertThat(actualOptions.getStsHeaderOverrides().get(TEST_HEADER_NAME_1), equalTo(TEST_HEADER_VALUE_1)); - assertThat(actualOptions.getStsHeaderOverrides(), hasKey(TEST_HEADER_NAME_2)); - assertThat(actualOptions.getStsHeaderOverrides().get(TEST_HEADER_NAME_2), equalTo(TEST_HEADER_VALUE_2)); + assertThat(actualOptions.getStsHeaderOverrides(), hasKey(headerName1)); + assertThat(actualOptions.getStsHeaderOverrides().get(headerName1), equalTo(headerValue1)); + assertThat(actualOptions.getStsHeaderOverrides(), hasKey(headerName2)); + assertThat(actualOptions.getStsHeaderOverrides().get(headerName2), equalTo(headerValue2)); } @Test void testCreateAllClients_WithValidHttpProxy_HostIP() throws IOException { + final Map metadata = generateConfigurationMetadata( + TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, false); final String testHttpProxy = "121.121.121.121:80"; - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(VALID_PROXY_IP); + metadata.put(PROXY_PARAMETER, testHttpProxy); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(metadata); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertEquals(connectionConfiguration.getProxy().get(), testHttpProxy); @@ -493,8 +568,11 @@ void testCreateAllClients_WithValidHttpProxy_HostIP() throws IOException { @Test void testCreateAllClients_WithValidHttpProxy_HostName() throws IOException { + final Map metadata = generateConfigurationMetadata( + TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, false); final String testHttpProxy = "example.com:80"; - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(VALID_PROXY_HOST_NAME); + metadata.put(PROXY_PARAMETER, testHttpProxy); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(metadata); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertEquals(connectionConfiguration.getProxy().get(), testHttpProxy); @@ -506,8 +584,11 @@ void testCreateAllClients_WithValidHttpProxy_HostName() throws IOException { @Test void testCreateAllClients_WithValidHttpProxy_SchemeProvided() throws IOException { + final Map metadata = generateConfigurationMetadata( + TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, false); final String testHttpProxy = "http://example.com:4350"; - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(VALID_HTTP_PROXY_SCHEME); + metadata.put(PROXY_PARAMETER, testHttpProxy); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(metadata); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertEquals(connectionConfiguration.getProxy().get(), testHttpProxy); @@ -518,9 +599,12 @@ void testCreateAllClients_WithValidHttpProxy_SchemeProvided() throws IOException } @Test - void testCreateClient_WithInvalidHttpProxy_InvalidPort() throws IOException { + void testCreateClient_WithInvalidHttpProxy_InvalidPort() throws JsonProcessingException { + final Map metadata = generateConfigurationMetadata( + TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, false); final String testHttpProxy = "example.com:port"; - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(INVALID_HTTP_PROXY_PORT); + metadata.put(PROXY_PARAMETER, testHttpProxy); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(metadata); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertEquals(connectionConfiguration.getProxy().get(), testHttpProxy); @@ -528,27 +612,36 @@ void testCreateClient_WithInvalidHttpProxy_InvalidPort() throws IOException { } @Test - void testCreateClient_WithInvalidHttpProxy_NoPort() throws IOException { + void testCreateClient_WithInvalidHttpProxy_NoPort() throws JsonProcessingException { + final Map metadata = generateConfigurationMetadata( + TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, false); final String testHttpProxy = "example.com"; - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(INVALID_HTTP_PROXY_NO_PORT); + metadata.put(PROXY_PARAMETER, testHttpProxy); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(metadata); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertThrows(IllegalArgumentException.class, () -> connectionConfiguration.createClient(awsCredentialsSupplier)); } @Test - void testCreateClient_WithInvalidHttpProxy_PortNotInRange() throws IOException { + void testCreateClient_WithInvalidHttpProxy_PortNotInRange() throws JsonProcessingException { + final Map metadata = generateConfigurationMetadata( + TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, false); final String testHttpProxy = "example.com:888888"; - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(INVALID_HTTP_PROXY_PORT_NOT_IN_RANGE); + metadata.put(PROXY_PARAMETER, testHttpProxy); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(metadata); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertThrows(IllegalArgumentException.class, () -> connectionConfiguration.createClient(awsCredentialsSupplier)); } @Test - void testCreateClient_WithInvalidHttpProxy_NotHttp() throws IOException { + void testCreateClient_WithInvalidHttpProxy_NotHttp() throws JsonProcessingException { + final Map metadata = generateConfigurationMetadata( + TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, false); final String testHttpProxy = "socket://example.com:port"; - final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(INVALID_HTTP_PROXY_NOT_HTTP); + metadata.put(PROXY_PARAMETER, testHttpProxy); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(metadata); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertEquals(connectionConfiguration.getProxy().get(), testHttpProxy); @@ -564,15 +657,64 @@ void testCreateClient_WithConnectionConfigurationBuilder_ProxyOptionalObjectShou assertNotNull(client); client.close(); } - - private OpenSearchSinkConfig generateOpenSearchSinkConfig(String pipelineName) throws IOException { - final File configurationFile = new File(getClass().getClassLoader().getResource(OPEN_SEARCH_SINK_CONFIGURATIONS).getFile()); - objectMapper = new ObjectMapper(new YAMLFactory()); - final Map pipelineConfigs = objectMapper.readValue(configurationFile, Map.class); - final Map pipelineConfig = (Map) pipelineConfigs.get(pipelineName); - final Map sinkMap = (Map) pipelineConfig.get("sink"); - final Map opensearchSinkMap = (Map) sinkMap.get("opensearch"); - String json = objectMapper.writeValueAsString(opensearchSinkMap); + + private OpenSearchSinkConfig generateOpenSearchSinkConfig( + final List hosts, final String username, final String password, + final Integer connectTimeout, final Integer socketTimeout, final boolean awsSigv4, final String awsRegion, + final String awsStsRoleArn, final String certPath, final boolean insecure) throws JsonProcessingException { + + final Map metadata = generateConfigurationMetadata(hosts, username, password, connectTimeout, socketTimeout, awsSigv4, awsRegion, awsStsRoleArn, certPath, insecure); + return getOpenSearchSinkConfigByConfigMetadata(metadata); + } + + private Map generateConfigurationMetadata( + final List hosts, final String username, final String password, + final Integer connectTimeout, final Integer socketTimeout, final boolean awsSigv4, final String awsRegion, + final String awsStsRoleArn, final String certPath, final boolean insecure) { + final Map metadata = new HashMap<>(); + metadata.put("hosts", hosts); + metadata.put("username", username); + metadata.put("password", password); + metadata.put("connect_timeout", connectTimeout); + metadata.put("socket_timeout", socketTimeout); + metadata.put("aws_sigv4", awsSigv4); + if (awsRegion != null) { + metadata.put("aws_region", awsRegion); + } + metadata.put("aws_sts_role_arn", awsStsRoleArn); + metadata.put("cert", certPath); + metadata.put("insecure", insecure); + return metadata; + } + + + private Map generateConfigurationMetadataWithAwsOption( + final List hosts, final String username, final String password, + final Integer connectTimeout, final Integer socketTimeout, final boolean serverless, final boolean awsSigv4, final String awsRegion, + final String awsStsRoleArn, final String awsStsExternalId, final String certPath, final boolean insecure, Map headerOverridesMap) { + final Map metadata = new HashMap<>(); + final Map awsOptionMetadata = new HashMap<>(); + metadata.put("hosts", hosts); + metadata.put("username", username); + metadata.put("password", password); + metadata.put("connect_timeout", connectTimeout); + metadata.put("socket_timeout", socketTimeout); + if (awsRegion != null) { + awsOptionMetadata.put("region", awsRegion); + } + awsOptionMetadata.put("serverless", serverless); + awsOptionMetadata.put("sts_role_arn", awsStsRoleArn); + awsOptionMetadata.put("sts_external_id", awsStsExternalId); + awsOptionMetadata.put("sts_header_overrides", headerOverridesMap); + metadata.put("aws", awsOptionMetadata); + metadata.put("cert", certPath); + metadata.put("insecure", insecure); + return metadata; + } + + private OpenSearchSinkConfig getOpenSearchSinkConfigByConfigMetadata(final Map metadata) throws JsonProcessingException { + objectMapper = new ObjectMapper(); + String json = new ObjectMapper().writeValueAsString(metadata); OpenSearchSinkConfig openSearchSinkConfig = objectMapper.readValue(json, OpenSearchSinkConfig.class); return openSearchSinkConfig; From 2ab0612740c736a22516fe5dd4e7ba6a7df37559 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Fri, 3 Jan 2025 14:53:47 -0800 Subject: [PATCH 51/62] yaml lines not needed conncection configuraion tests switched back to metadata approach Signed-off-by: Maxwell Brown --- .../open-search-sink-configurations.yaml | 179 +----------------- 1 file changed, 1 insertion(+), 178 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/test/resources/open-search-sink-configurations.yaml b/data-prepper-plugins/opensearch/src/test/resources/open-search-sink-configurations.yaml index c68caba687..2ccd596dbc 100644 --- a/data-prepper-plugins/opensearch/src/test/resources/open-search-sink-configurations.yaml +++ b/data-prepper-plugins/opensearch/src/test/resources/open-search-sink-configurations.yaml @@ -118,181 +118,4 @@ dlq-file-and-dlq-plugin: bucket: "my-dlq-bucket" key_path_prefix: "dlq-files/" dlq_file: foo.txt - max_retries: 10 -empty-sink: - sink: - opensearch: - hosts: [ "http://localhost:9200" ] -aws-serverless-default: - sink: - opensearch: - hosts: [ "http://localhost:9200" ] - cert: "./src/test/resources/test-ca.pem" - aws: - serverless: true - region: "us-east-2" - sts_role_arn: "arn:aws:iam::123456789012:role/test-role" -es6-default: - sink: - opensearch: - hosts: [ "http://localhost:9200" ] - distribution_version: "es6" -aws-serverless-no-cert: - sink: - opensearch: - hosts: [ "http://localhost:9200" ] - aws: - serverless: true - region: "us-east-2" - sts_role_arn: "arn:aws:iam::123456789012:role/test-role" -basic-credentials-no-cert: - sink: - opensearch: - hosts: [ "http://localhost:9200" ] - username: "test-username" - password: "test-password" - connect_timeout: 5 - socket_timeout: 10 -basic-credentials-no-cert-insecure: - sink: - opensearch: - hosts: [ "http://localhost:9200" ] - username: "test-username" - password: "test-password" - connect_timeout: 5 - socket_timeout: 10 - insecure: true -basic-credentials-with-cert: - sink: - opensearch: - hosts: [ "http://localhost:9200" ] - username: "test-username" - password: "test-password" - cert: "./src/test/resources/test-ca.pem" - connect_timeout: 5 - socket_timeout: 10 -aws-region-only: - sink: - opensearch: - hosts: [ "http://localhost:9200" ] - aws: - region: "us-east-2" -serverless-options: - sink: - opensearch: - hosts: [ "http://localhost:9200" ] - username: "test-username" - password: "test-password" - connect_timeout: 5 - socket_timeout: 10 - aws: - serverless: true - region: "us-east-2" - sts_role_arn: "arn:aws:iam::123456789012:role/test-role" - serverless_options: - network_policy_name: "test network policy" - collection_name: "test collection" - vpce_id: "test vpce id" -aws-region-only-insecure: - sink: - opensearch: - hosts: [ "http://localhost:9200" ] - aws: - region: "us-east-1" - insecure: true -aws-with-cert: - sink: - opensearch: - hosts: [ "http://localhost:9200" ] - aws: - region: "us-east-1" - cert: "./src/test/resources/test-ca.pem" -aws-with-cert-and-arn: - sink: - opensearch: - hosts: [ "http://localhost:9200" ] - aws: - region: "us-east-1" - sts_role_arn: "arn:aws:iam::123456789012:role/test-role" - cert: "./src/test/resources/test-ca.pem" -aws-with-2-header: - sink: - opensearch: - hosts: [ "http://localhost:9200" ] - aws: - region: "us-east-1" - sts_role_arn: "arn:aws:iam::123456789012:role/test-role" - sts_external_id: "test-external-id" - sts_header_overrides: - header1: "test-header-1" - header2: "test-header-2" -valid-proxy-ip: - sink: - opensearch: - hosts: [ "http://localhost:9200" ] - username: "test-username" - password: "test-password" - connect_timeout: 5 - socket_timeout: 10 - cert: "./src/test/resources/test-ca.pem" - proxy: "121.121.121.121:80" -valid-proxy-host-name: - sink: - opensearch: - hosts: [ "http://localhost:9200" ] - username: "test-username" - password: "test-password" - connect_timeout: 5 - socket_timeout: 10 - cert: "./src/test/resources/test-ca.pem" - proxy: "example.com:80" -valid-http-proxy-scheme: - sink: - opensearch: - hosts: [ "http://localhost:9200" ] - username: "test-username" - password: "test-password" - connect_timeout: 5 - socket_timeout: 10 - cert: "./src/test/resources/test-ca.pem" - proxy: "http://example.com:4350" -invalid-http-proxy-port: - sink: - opensearch: - hosts: [ "http://localhost:9200" ] - username: "test-username" - password: "test-password" - connect_timeout: 5 - socket_timeout: 10 - cert: "./src/test/resources/test-ca.pem" - proxy: "example.com:port" -invalid-http-proxy-no-port: - sink: - opensearch: - hosts: [ "http://localhost:9200" ] - username: "test-username" - password: "test-password" - connect_timeout: 5 - socket_timeout: 10 - cert: "./src/test/resources/test-ca.pem" - proxy: "example.com" -invalid-http-proxy-port-not-in-range: - sink: - opensearch: - hosts: [ "http://localhost:9200" ] - username: "test-username" - password: "test-password" - connect_timeout: 5 - socket_timeout: 10 - cert: "./src/test/resources/test-ca.pem" - proxy: "example.com:888888" -invalid-http-proxy-not-http: - sink: - opensearch: - hosts: [ "http://localhost:9200" ] - username: "test-username" - password: "test-password" - connect_timeout: 5 - socket_timeout: 10 - cert: "./src/test/resources/test-ca.pem" - proxy: "socket://example.com:port" \ No newline at end of file + max_retries: 10 \ No newline at end of file From 0f5e6f1be29a609d0aae8a0520528f576cf8ef02 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Mon, 6 Jan 2025 09:02:26 -0800 Subject: [PATCH 52/62] style fix Signed-off-by: Maxwell Brown --- .../dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index 2e83534ba0..b87c5f0c34 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -888,7 +888,8 @@ public void testOpenSearchBulkActionsCreateWithExpression() throws IOException, Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); Event event = (Event) testRecords.get(0).getData(); - event.getMetadata().setAttribute("action", "create");metadata.put(IndexConfiguration.ACTION, "create"); + event.getMetadata().setAttribute("action", "create"); + metadata.put(IndexConfiguration.ACTION, "create"); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); From 21b2b97c133909bc7c7e57f46ea62aa87bd5b61e Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Mon, 6 Jan 2025 09:31:19 -0800 Subject: [PATCH 53/62] assert True issue Signed-off-by: Maxwell Brown --- .../configuration/OpenSearchSinkConfig.java | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java index 1b21f63f68..8c29d12d64 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java @@ -202,15 +202,6 @@ public boolean isNotAwsSigv4AndAwsOption() { @Size(min = 1, message = "Region cannot be empty string") private String awsRegion = DEFAULT_AWS_REGION; - @Deprecated - @AssertTrue(message = "aws_region option cannot be used along with aws option. It is preferred to only use aws option as aws_region is deprecated.") - public boolean isNotAwsRegionAndAwsOption() { - if (awsAuthenticationOptions != null && awsRegion != null) { - return false; - } - return true; - } - @Deprecated @Getter @JsonProperty("aws_sts_role_arn") @@ -302,7 +293,5 @@ public boolean isNotDocumentIdFieldAndDocumentId() { @Getter @JsonProperty("routing_field") private String routingField = null; - - } From 2e2c58683d30955b7c4e3247f00b6b22d21b3380 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Tue, 7 Jan 2025 10:12:57 -0800 Subject: [PATCH 54/62] put routing field test back Signed-off-by: Maxwell Brown --- .../sink/opensearch/OpenSearchSinkTest.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java index 815698a123..59019e0b0a 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java @@ -259,6 +259,21 @@ void doOutput_with_invalid_version_expression_catches_NumberFormatException_and_ verify(dynamicDocumentVersionDroppedEvents).increment(); } + @Test + void test_routing_field_in_document() throws IOException { + String routingFieldKey = UUID.randomUUID().toString(); + String routingKey = UUID.randomUUID().toString(); + String routingFieldValue = UUID.randomUUID().toString(); + when(indexConfiguration.getRoutingField()).thenReturn(routingFieldKey); + when(indexConfiguration.getRouting()).thenReturn(routingKey); + final OpenSearchSink objectUnderTest = createObjectUnderTest(); + final Event event = JacksonEvent.builder() + .withEventType("event") + .withData(Collections.singletonMap(routingFieldKey, routingFieldValue)) + .build(); + assertThat(objectUnderTest.getDocument(event).getRoutingField(), equalTo(Optional.of(routingFieldValue))); + } + @Test void test_routing_in_document() throws IOException { String routingValue = UUID.randomUUID().toString(); From 0eeada66111df9c2abe9adf0880bb4f7f958aa5d Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Wed, 8 Jan 2025 12:29:50 -0800 Subject: [PATCH 55/62] add license header to new files Signed-off-by: Maxwell Brown --- .../sink/opensearch/configuration/ActionConfiguration.java | 5 +++++ .../sink/opensearch/configuration/DlqConfiguration.java | 5 +++++ .../sink/opensearch/configuration/OpenSearchSinkConfig.java | 5 +++++ .../sink/opensearch/configuration/ServerlessOptions.java | 5 +++++ 4 files changed, 20 insertions(+) diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ActionConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ActionConfiguration.java index f867db8c50..02c92ea609 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ActionConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ActionConfiguration.java @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.dataprepper.plugins.sink.opensearch.configuration; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/DlqConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/DlqConfiguration.java index ea82e33cd3..2ba4f9805d 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/DlqConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/DlqConfiguration.java @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.dataprepper.plugins.sink.opensearch.configuration; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java index 8c29d12d64..d30c448db9 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.dataprepper.plugins.sink.opensearch.configuration; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ServerlessOptions.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ServerlessOptions.java index a274f47519..2899dfd419 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ServerlessOptions.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ServerlessOptions.java @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.dataprepper.plugins.sink.opensearch.configuration; import com.fasterxml.jackson.annotation.JsonProperty; From 7edfe65c4c8ed418fcb7f450b61f65809c3d50e9 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Wed, 8 Jan 2025 12:43:56 -0800 Subject: [PATCH 56/62] move build.gradle changes to data-prepper-plugins/opensearch/build.gradle Signed-off-by: Maxwell Brown --- build.gradle | 2 -- data-prepper-plugins/opensearch/build.gradle | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 3360da81b5..7f78c2424f 100644 --- a/build.gradle +++ b/build.gradle @@ -69,8 +69,6 @@ subprojects { } } dependencies { - compileOnly 'org.projectlombok:lombok:1.18.30' - annotationProcessor 'org.projectlombok:lombok:1.18.30' implementation platform('com.fasterxml.jackson:jackson-bom:2.17.2') implementation platform('org.eclipse.jetty:jetty-bom:9.4.53.v20231009') implementation platform('io.micrometer:micrometer-bom:1.10.5') diff --git a/data-prepper-plugins/opensearch/build.gradle b/data-prepper-plugins/opensearch/build.gradle index 5e7879d8d1..2d02542c37 100644 --- a/data-prepper-plugins/opensearch/build.gradle +++ b/data-prepper-plugins/opensearch/build.gradle @@ -4,6 +4,8 @@ */ dependencies { + compileOnly 'org.projectlombok:lombok:1.18.30' + annotationProcessor 'org.projectlombok:lombok:1.18.30' implementation project(':data-prepper-api') implementation libs.armeria.core testImplementation project(':data-prepper-api').sourceSets.test.output From 3d5d8c81dc1404c2d21c9175931b6c0a7deb6582 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Thu, 16 Jan 2025 10:49:14 -0800 Subject: [PATCH 57/62] update actions artifacts version for integration tests Signed-off-by: Maxwell Brown --- .github/workflows/gradle.yml | 4 ++-- .../opensearch-sink-opendistro-integration-tests.yml | 4 ++-- .../opensearch-sink-opensearch-integration-tests.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index cb4373ee24..cc495c596d 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -28,7 +28,7 @@ jobs: run: ./gradlew --parallel --max-workers 2 build - name: Upload Unit Test Results if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: data-prepper-test-results-java-${{ matrix.java }} path: '**/test-results/**/*.xml' @@ -45,7 +45,7 @@ jobs: steps: - name: Download Artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: test-results diff --git a/.github/workflows/opensearch-sink-opendistro-integration-tests.yml b/.github/workflows/opensearch-sink-opendistro-integration-tests.yml index 6f4ac3d457..42d508ce7e 100644 --- a/.github/workflows/opensearch-sink-opendistro-integration-tests.yml +++ b/.github/workflows/opensearch-sink-opendistro-integration-tests.yml @@ -43,7 +43,7 @@ jobs: ./gradlew :data-prepper-plugins:opensearch:integrationTest -Dtests.opensearch.host=localhost:9200 -Dtests.opensearch.user=admin -Dtests.opensearch.password=admin -Dtests.opensearch.bundle=true -Dtests.opensearch.version=opendistro:${{ matrix.opendistro }} - name: Upload Unit Test Results if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: data-prepper-opensearch-integration-tests-opendistro-${{ matrix.opendistro }}-java-${{ matrix.java }} path: '**/test-results/**/*.xml' @@ -56,7 +56,7 @@ jobs: steps: - name: Download Artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: test-results diff --git a/.github/workflows/opensearch-sink-opensearch-integration-tests.yml b/.github/workflows/opensearch-sink-opensearch-integration-tests.yml index dc18aec096..be8bb25fde 100644 --- a/.github/workflows/opensearch-sink-opensearch-integration-tests.yml +++ b/.github/workflows/opensearch-sink-opensearch-integration-tests.yml @@ -43,7 +43,7 @@ jobs: ./gradlew :data-prepper-plugins:opensearch:integrationTest -Dtests.opensearch.host=localhost:9200 -Dtests.opensearch.user=admin -Dtests.opensearch.password=admin -Dtests.opensearch.bundle=true -Dtests.opensearch.version=opensearch:${{ matrix.opensearch }} - name: Upload Unit Test Results if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: data-prepper-opensearch-integration-tests-opensearch-${{ matrix.opensearch }}-java-${{ matrix.java }} path: '**/test-results/**/*.xml' @@ -56,7 +56,7 @@ jobs: steps: - name: Download Artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: test-results From 2c78c46b6fb1e1fe0d41aeccbc828ae27c90e8ad Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Thu, 16 Jan 2025 10:59:03 -0800 Subject: [PATCH 58/62] fixed new test Signed-off-by: Maxwell Brown --- .../plugins/sink/opensearch/ConnectionConfigurationTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java index 3c3dcd6d74..a2356ccb94 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java @@ -258,10 +258,10 @@ void testCreateClientWithCertPath() throws IOException { @Test void testCreateClientWithInsecureAndCertPath() throws IOException { // Insecure should take precedence over cert path when both are set - final PluginSetting pluginSetting = generatePluginSetting( + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, true); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertNull(connectionConfiguration.getCertPath()); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); assertNotNull(client); From 7ff3d0007da06f4401ed31e3c0e67c2f1b9e61c0 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Fri, 17 Jan 2025 16:13:27 -0800 Subject: [PATCH 59/62] FailedbulkOperationConveter takes the same input twice Signed-off-by: Maxwell Brown --- .../plugins/sink/opensearch/OpenSearchSink.java | 3 +-- .../sink/opensearch/dlq/FailedBulkOperationConverter.java | 7 ++----- .../opensearch/dlq/FailedBulkOperationConverterTest.java | 6 ++---- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java index ab111f9b01..fa28cadc81 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java @@ -180,8 +180,7 @@ public OpenSearchSink(final PluginSetting pluginSetting, this.versionType = openSearchSinkConfig.getIndexConfiguration().getVersionType(); this.versionExpression = openSearchSinkConfig.getIndexConfiguration().getVersionExpression(); this.indexManagerFactory = new IndexManagerFactory(new ClusterSettingsParser()); - this.failedBulkOperationConverter = new FailedBulkOperationConverter(pipeline, PLUGIN_NAME, - PLUGIN_NAME); + this.failedBulkOperationConverter = new FailedBulkOperationConverter(pipeline, PLUGIN_NAME); this.initialized = false; this.lock = new ReentrantLock(true); this.bulkRequestMap = new ConcurrentHashMap<>(); diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/dlq/FailedBulkOperationConverter.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/dlq/FailedBulkOperationConverter.java index 31b24fec8f..6c2e91b5b3 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/dlq/FailedBulkOperationConverter.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/dlq/FailedBulkOperationConverter.java @@ -19,15 +19,12 @@ public class FailedBulkOperationConverter { private final String pluginName; private final String pipelineName; - private final String pluginId; - public FailedBulkOperationConverter(final String pipelineName, final String pluginName, final String pluginId) { + public FailedBulkOperationConverter(final String pipelineName, final String pluginName) { Objects.requireNonNull(pipelineName); - Objects.requireNonNull(pluginId); Objects.requireNonNull(pluginName); this.pluginName = pluginName; this.pipelineName = pipelineName; - this.pluginId = pluginId; } public DlqObject convertToDlqObject(final FailedBulkOperation failedBulkOperation) { @@ -54,7 +51,7 @@ public DlqObject convertToDlqObject(final FailedBulkOperation failedBulkOperatio .withFailedData(failedDlqDataBuilder.build()) .withPluginName(pluginName) .withPipelineName(pipelineName) - .withPluginId(pluginId) + .withPluginId(pluginName) .withEventHandle(bulkOperationWithHandle.getEventHandle()) .build(); } diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/dlq/FailedBulkOperationConverterTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/dlq/FailedBulkOperationConverterTest.java index 17f0e52079..f02e39fd96 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/dlq/FailedBulkOperationConverterTest.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/dlq/FailedBulkOperationConverterTest.java @@ -31,7 +31,6 @@ public class FailedBulkOperationConverterTest { private String testId; private String pipelineName; private String pluginName; - private String pluginId; private BulkOperation bulkOperation; private BulkResponseItem bulkResponseItem; private String errorReason; @@ -47,7 +46,6 @@ public void setup() { testId = UUID.randomUUID().toString(); pipelineName = UUID.randomUUID().toString(); pluginName = UUID.randomUUID().toString(); - pluginId = UUID.randomUUID().toString(); errorReason = UUID.randomUUID().toString(); failureMessage = UUID.randomUUID().toString(); @@ -75,7 +73,7 @@ public void setup() { when(bulkResponseItem.error()).thenReturn(errorCause); when(errorCause.reason()).thenReturn(errorReason); - converter = new FailedBulkOperationConverter(pipelineName, pluginName, pluginId); + converter = new FailedBulkOperationConverter(pipelineName, pluginName); } @Test @@ -136,7 +134,7 @@ public void testConvertToDlqObjectWithOnlyBulkResponseItem() { private void validateResponse(final DlqObject result, final String expectedErrorMessage) { assertThat(result, is(notNullValue())); assertThat(result.getPipelineName(), is(equalTo(pipelineName))); - assertThat(result.getPluginId(), is(equalTo(pluginId))); + assertThat(result.getPluginId(), is(equalTo(pluginName))); assertThat(result.getPluginName(), is(equalTo(pluginName))); final Object failedData = result.getFailedData(); assertThat(failedData, is(notNullValue())); From c7ac019cc245e59a2b74e11647a16530d7a0f549 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Wed, 22 Jan 2025 10:27:24 -0800 Subject: [PATCH 60/62] readd a routing field test Signed-off-by: Maxwell Brown --- .../opensearch/index/IndexConfigurationTests.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfigurationTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfigurationTests.java index c3c59abcd8..6ba7cbaffc 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfigurationTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfigurationTests.java @@ -53,6 +53,7 @@ import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.DOCUMENT_ROOT_KEY; import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.DOCUMENT_VERSION_EXPRESSION; import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.ROUTING; +import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.ROUTING_FIELD; import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.SERVERLESS; import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.TEMPLATE_TYPE; import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConstants.RAW_DEFAULT_TEMPLATE_FILE; @@ -491,6 +492,17 @@ public void testReadIndexConfig_routing() throws JsonProcessingException { assertEquals(expectedRoutingValue, indexConfiguration.getRouting()); } + @Test + public void testReadIndexConfig_routingField() throws JsonProcessingException { + final Map metadata = initializeConfigMetaData( + IndexType.CUSTOM.getValue(), "foo", null, null, null, null, null); + final String expectedRoutingFieldValue = UUID.randomUUID().toString(); + metadata.put(ROUTING_FIELD, expectedRoutingFieldValue); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metadata); + final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig); + assertEquals(expectedRoutingFieldValue, indexConfiguration.getRoutingField()); + } + @ParameterizedTest @ValueSource(strings = {"${key}", "${getMetadata(\"key\")}"}) public void testReadIndexConfig_withValidDocumentVersionExpression(final String versionExpression) throws JsonProcessingException { From 122aabaec970677db187bb0fb432c8134bccf20c Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Thu, 23 Jan 2025 08:51:56 -0800 Subject: [PATCH 61/62] remove refernces to invalid authentication configuration Signed-off-by: Maxwell Brown --- .../opensearch/ConnectionConfiguration.java | 30 +------ .../opensearch/OpenSearchClientRefresher.java | 18 +--- .../OpenSearchClientRefresherTest.java | 90 +------------------ 3 files changed, 11 insertions(+), 127 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java index 6debecd86a..4e3aa49c5a 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java @@ -32,8 +32,6 @@ import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.AwsAuthenticationConfiguration; import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig; import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.ServerlessOptions; - -import org.opensearch.dataprepper.plugins.source.opensearch.AuthConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.arns.Arn; @@ -100,7 +98,6 @@ public class ConnectionConfiguration { private final String serverlessCollectionName; private final String serverlessVpceId; private final boolean requestCompressionEnabled; - private final AuthConfig authConfig; List getHosts() { return hosts; @@ -162,10 +159,6 @@ boolean isRequestCompressionEnabled() { return requestCompressionEnabled; } - public AuthConfig getAuthConfig() { - return authConfig; - } - private ConnectionConfiguration(final Builder builder) { this.hosts = builder.hosts; this.username = builder.username; @@ -185,7 +178,6 @@ private ConnectionConfiguration(final Builder builder) { this.serverlessCollectionName = builder.serverlessCollectionName; this.serverlessVpceId = builder.serverlessVpceId; this.requestCompressionEnabled = builder.requestCompressionEnabled; - this.authConfig = builder.authConfig; } public static ConnectionConfiguration readConnectionConfiguration(final OpenSearchSinkConfig openSearchSinkConfig){ @@ -322,18 +314,10 @@ public AwsCredentialsOptions createAwsCredentialsOptions() { private void attachUserCredentials(final RestClientBuilder restClientBuilder) { final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - if (authConfig != null) { - if (authConfig.getUsername() != null) { - LOG.info("Using the authentication provided in the config."); - credentialsProvider.setCredentials( - AuthScope.ANY, new UsernamePasswordCredentials(authConfig.getUsername(), authConfig.getPassword())); - } - } else { - if (username != null) { - LOG.info("Using the username provided in the config."); - credentialsProvider.setCredentials( - AuthScope.ANY, new UsernamePasswordCredentials(username, password)); - } + if (username != null) { + LOG.info("Using the username provided in the config."); + credentialsProvider.setCredentials( + AuthScope.ANY, new UsernamePasswordCredentials(username, password)); } restClientBuilder.setHttpClientConfigCallback( httpClientBuilder -> { @@ -497,7 +481,6 @@ public static class Builder { private String serverlessCollectionName; private String serverlessVpceId; private boolean requestCompressionEnabled; - private AuthConfig authConfig; private void validateStsRoleArn(final String awsStsRoleArn) { final Arn arn = getArn(awsStsRoleArn); @@ -622,11 +605,6 @@ public Builder withRequestCompressionEnabled(final boolean requestCompressionEna return this; } - public Builder withAuthConfig(final AuthConfig authConfig) { - this.authConfig = authConfig; - return this; - } - public ConnectionConfiguration build() { return new ConnectionConfiguration(this); } diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresher.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresher.java index bd3b268e2d..2dd987b8fd 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresher.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresher.java @@ -74,23 +74,13 @@ public void update(OpenSearchSinkConfig openSearchSinkConfig) { private boolean basicAuthChanged(final ConnectionConfiguration newConfig) { final String existingUsername; final String existingPassword; - if (currentConfig.getAuthConfig() != null) { - existingUsername = currentConfig.getAuthConfig().getUsername(); - existingPassword = currentConfig.getAuthConfig().getPassword(); - } else { - existingUsername = currentConfig.getUsername(); - existingPassword = currentConfig.getPassword(); - } + existingUsername = currentConfig.getUsername(); + existingPassword = currentConfig.getPassword(); final String newUsername; final String newPassword; - if (newConfig.getAuthConfig() != null) { - newUsername = newConfig.getAuthConfig().getUsername(); - newPassword = newConfig.getAuthConfig().getPassword(); - } else { - newUsername = newConfig.getUsername(); - newPassword = newConfig.getPassword(); - } + newUsername = newConfig.getUsername(); + newPassword = newConfig.getPassword(); return !Objects.equals(existingUsername, newUsername) || !Objects.equals(existingPassword, newPassword); diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresherTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresherTest.java index 24c27bed89..e31c680e8d 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresherTest.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresherTest.java @@ -10,7 +10,6 @@ import org.opensearch.client.opensearch.OpenSearchClient; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig; -import org.opensearch.dataprepper.plugins.source.opensearch.AuthConfig; import java.util.function.Function; @@ -41,9 +40,6 @@ class OpenSearchClientRefresherTest { @Mock private OpenSearchClient openSearchClient; - @Mock - private AuthConfig authConfig; - @Mock private PluginMetrics pluginMetrics; @@ -69,7 +65,7 @@ void testGet() { } @Test - void testGetAfterUpdateWithDeprecatedBasicAuthUnchanged() { + void testGetAfterUpdateWithBasicAuthUnchanged() { final OpenSearchClientRefresher objectUnderTest = createObjectUnderTest(); assertThat(objectUnderTest.get(), equalTo(openSearchClient)); verify(clientFunction, times(1)).apply(any()); @@ -90,31 +86,7 @@ void testGetAfterUpdateWithDeprecatedBasicAuthUnchanged() { } @Test - void testGetAfterUpdateWithBasicAuthUnchanged() { - final OpenSearchClientRefresher objectUnderTest = createObjectUnderTest(); - assertThat(objectUnderTest.get(), equalTo(openSearchClient)); - verify(clientFunction, times(1)).apply(any()); - when(connectionConfiguration.getAuthConfig()).thenReturn(authConfig); - when(authConfig.getUsername()).thenReturn(TEST_USERNAME); - when(authConfig.getPassword()).thenReturn(TEST_PASSWORD); - final OpenSearchSinkConfig newConfig = mock(OpenSearchSinkConfig.class); - final ConnectionConfiguration newConnectionConfiguration = mock(ConnectionConfiguration.class); - final AuthConfig newAuthConfig = mock(AuthConfig.class); - when(newConnectionConfiguration.getAuthConfig()).thenReturn(newAuthConfig); - when(newAuthConfig.getUsername()).thenReturn(TEST_USERNAME); - when(newAuthConfig.getPassword()).thenReturn(TEST_PASSWORD); - try (MockedStatic configurationMockedStatic = mockStatic( - ConnectionConfiguration.class)) { - configurationMockedStatic.when(() -> ConnectionConfiguration.readConnectionConfiguration(eq(newConfig))) - .thenReturn(newConnectionConfiguration); - objectUnderTest.update(newConfig); - } - assertThat(objectUnderTest.get(), equalTo(openSearchClient)); - verifyNoMoreInteractions(clientFunction); - } - - @Test - void testGetAfterUpdateWithDeprecatedUsernameChanged() { + void testGetAfterUpdateWithUsernameChanged() { when(pluginMetrics.counter(CREDENTIALS_CHANGED)).thenReturn(credentialsChangeCounter); final OpenSearchClientRefresher objectUnderTest = createObjectUnderTest(); assertThat(objectUnderTest.get(), equalTo(openSearchClient)); @@ -137,36 +109,9 @@ void testGetAfterUpdateWithDeprecatedUsernameChanged() { verify(clientFunction, times(2)).apply(any()); } - @Test - void testGetAfterUpdateWithUsernameChanged() { - when(pluginMetrics.counter(CREDENTIALS_CHANGED)).thenReturn(credentialsChangeCounter); - final OpenSearchClientRefresher objectUnderTest = createObjectUnderTest(); - assertThat(objectUnderTest.get(), equalTo(openSearchClient)); - verify(clientFunction, times(1)).apply(any()); - assertThat(objectUnderTest.get(), equalTo(openSearchClient)); - when(connectionConfiguration.getAuthConfig()).thenReturn(authConfig); - when(authConfig.getUsername()).thenReturn(TEST_USERNAME); - when(authConfig.getPassword()).thenReturn(TEST_PASSWORD); - final OpenSearchSinkConfig newConfig = mock(OpenSearchSinkConfig.class); - final ConnectionConfiguration newConnectionConfiguration = mock(ConnectionConfiguration.class); - final AuthConfig newAuthConfig = mock(AuthConfig.class); - when(newConnectionConfiguration.getAuthConfig()).thenReturn(newAuthConfig); - when(newAuthConfig.getUsername()).thenReturn(TEST_USERNAME + "_changed"); - final OpenSearchClient newClient = mock(OpenSearchClient.class); - when(clientFunction.apply(eq(newConnectionConfiguration))).thenReturn(newClient); - try (MockedStatic configurationMockedStatic = mockStatic( - ConnectionConfiguration.class)) { - configurationMockedStatic.when(() -> ConnectionConfiguration.readConnectionConfiguration(eq(newConfig))) - .thenReturn(newConnectionConfiguration); - objectUnderTest.update(newConfig); - } - assertThat(objectUnderTest.get(), equalTo(newClient)); - verify(credentialsChangeCounter).increment(); - verify(clientFunction, times(2)).apply(any()); - } @Test - void testGetAfterUpdateWithDeprecatedPasswordChanged() { + void testGetAfterUpdateWithPasswordChanged() { when(pluginMetrics.counter(CREDENTIALS_CHANGED)).thenReturn(credentialsChangeCounter); final OpenSearchClientRefresher objectUnderTest = createObjectUnderTest(); assertThat(objectUnderTest.get(), equalTo(openSearchClient)); @@ -191,35 +136,6 @@ void testGetAfterUpdateWithDeprecatedPasswordChanged() { verify(clientFunction, times(2)).apply(any()); } - @Test - void testGetAfterUpdateWithPasswordChanged() { - when(pluginMetrics.counter(CREDENTIALS_CHANGED)).thenReturn(credentialsChangeCounter); - final OpenSearchClientRefresher objectUnderTest = createObjectUnderTest(); - assertThat(objectUnderTest.get(), equalTo(openSearchClient)); - verify(clientFunction, times(1)).apply(any()); - assertThat(objectUnderTest.get(), equalTo(openSearchClient)); - when(connectionConfiguration.getAuthConfig()).thenReturn(authConfig); - when(authConfig.getUsername()).thenReturn(TEST_USERNAME); - when(authConfig.getPassword()).thenReturn(TEST_PASSWORD); - final OpenSearchSinkConfig newConfig = mock(OpenSearchSinkConfig.class); - final ConnectionConfiguration newConnectionConfiguration = mock(ConnectionConfiguration.class); - final AuthConfig newAuthConfig = mock(AuthConfig.class); - when(newConnectionConfiguration.getAuthConfig()).thenReturn(newAuthConfig); - when(newAuthConfig.getUsername()).thenReturn(TEST_USERNAME); - when(newAuthConfig.getPassword()).thenReturn(TEST_PASSWORD + "_changed"); - final OpenSearchClient newClient = mock(OpenSearchClient.class); - when(clientFunction.apply(eq(newConnectionConfiguration))).thenReturn(newClient); - try (MockedStatic configurationMockedStatic = mockStatic( - ConnectionConfiguration.class)) { - configurationMockedStatic.when(() -> ConnectionConfiguration.readConnectionConfiguration(eq(newConfig))) - .thenReturn(newConnectionConfiguration); - objectUnderTest.update(newConfig); - } - assertThat(objectUnderTest.get(), equalTo(newClient)); - verify(credentialsChangeCounter).increment(); - verify(clientFunction, times(2)).apply(any()); - } - @Test void testGetAfterUpdateClientFailure() { when(pluginMetrics.counter(CREDENTIALS_CHANGED)).thenReturn(credentialsChangeCounter); From faa5c179183c44824c2b7cdc37d38eb77182f146 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Fri, 24 Jan 2025 15:00:20 -0800 Subject: [PATCH 62/62] we want to support authentication: username: password: config Signed-off-by: Maxwell Brown --- .../sink/opensearch/OpenSearchSinkIT.java | 3 +- .../opensearch/ConnectionConfiguration.java | 44 +++++++-- .../opensearch/OpenSearchClientRefresher.java | 18 +++- .../opensearch/configuration/AuthConfig.java | 19 ++++ .../configuration/OpenSearchSinkConfig.java | 9 ++ .../ConnectionConfigurationTests.java | 28 ++++++ .../OpenSearchClientRefresherTest.java | 92 ++++++++++++++++++- 7 files changed, 194 insertions(+), 19 deletions(-) create mode 100644 data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AuthConfig.java diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index b87c5f0c34..9df9095fd4 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -1670,8 +1670,7 @@ private Map initializeConfigurationMetadata(final String indexTy final String user = System.getProperty("tests.opensearch.user"); final String password = System.getProperty("tests.opensearch.password"); if (user != null) { - metadata.put(USERNAME, user); - metadata.put(PASSWORD, password); + metadata.put(AUTHENTICATION, Map.of(USERNAME, user, PASSWORD, password)); } final String distributionVersion = DeclaredOpenSearchVersion.OPENDISTRO_0_10.compareTo( OpenSearchIntegrationHelper.getVersion()) >= 0 ? diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java index 4e3aa49c5a..d16499c45c 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java @@ -29,6 +29,7 @@ import org.opensearch.dataprepper.aws.api.AwsCredentialsSupplier; import org.opensearch.dataprepper.aws.api.AwsRequestSigningApache4Interceptor; import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.PreSerializedJsonpMapper; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.AuthConfig; import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.AwsAuthenticationConfiguration; import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig; import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.ServerlessOptions; @@ -98,6 +99,7 @@ public class ConnectionConfiguration { private final String serverlessCollectionName; private final String serverlessVpceId; private final boolean requestCompressionEnabled; + private final AuthConfig authConfig; List getHosts() { return hosts; @@ -159,6 +161,10 @@ boolean isRequestCompressionEnabled() { return requestCompressionEnabled; } + public AuthConfig getAuthConfig() { + return authConfig; + } + private ConnectionConfiguration(final Builder builder) { this.hosts = builder.hosts; this.username = builder.username; @@ -178,6 +184,7 @@ private ConnectionConfiguration(final Builder builder) { this.serverlessCollectionName = builder.serverlessCollectionName; this.serverlessVpceId = builder.serverlessVpceId; this.requestCompressionEnabled = builder.requestCompressionEnabled; + this.authConfig = builder.authConfig; } public static ConnectionConfiguration readConnectionConfiguration(final OpenSearchSinkConfig openSearchSinkConfig){ @@ -185,11 +192,16 @@ public static ConnectionConfiguration readConnectionConfiguration(final OpenSear ConnectionConfiguration.Builder builder = new ConnectionConfiguration.Builder(hosts); final String username = openSearchSinkConfig.getUsername(); final String password = openSearchSinkConfig.getPassword(); - if (username != null) { - builder = builder.withUsername(username); - } - if (password != null) { - builder = builder.withPassword(password); + final AuthConfig authConfig = openSearchSinkConfig.getAuthConfig(); + if (authConfig != null) { + builder = builder.withAuthConfig(authConfig); + } else { + if (username != null) { + builder = builder.withUsername(username); + } + if (password != null) { + builder = builder.withPassword(password); + } } final Integer socketTimeout = openSearchSinkConfig.getSocketTimeout(); if (socketTimeout != null) { @@ -314,10 +326,18 @@ public AwsCredentialsOptions createAwsCredentialsOptions() { private void attachUserCredentials(final RestClientBuilder restClientBuilder) { final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - if (username != null) { - LOG.info("Using the username provided in the config."); - credentialsProvider.setCredentials( - AuthScope.ANY, new UsernamePasswordCredentials(username, password)); + if (authConfig != null) { + if (authConfig.getUsername() != null) { + LOG.info("Using the authentication provided in the config."); + credentialsProvider.setCredentials( + AuthScope.ANY, new UsernamePasswordCredentials(authConfig.getUsername(), authConfig.getPassword())); + } + } else { + if (username != null) { + LOG.info("Using the username provided in the config."); + credentialsProvider.setCredentials( + AuthScope.ANY, new UsernamePasswordCredentials(username, password)); + } } restClientBuilder.setHttpClientConfigCallback( httpClientBuilder -> { @@ -481,6 +501,7 @@ public static class Builder { private String serverlessCollectionName; private String serverlessVpceId; private boolean requestCompressionEnabled; + private AuthConfig authConfig; private void validateStsRoleArn(final String awsStsRoleArn) { final Arn arn = getArn(awsStsRoleArn); @@ -605,6 +626,11 @@ public Builder withRequestCompressionEnabled(final boolean requestCompressionEna return this; } + public Builder withAuthConfig(final AuthConfig authConfig) { + this.authConfig = authConfig; + return this; + } + public ConnectionConfiguration build() { return new ConnectionConfiguration(this); } diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresher.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresher.java index 2dd987b8fd..bd3b268e2d 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresher.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresher.java @@ -74,13 +74,23 @@ public void update(OpenSearchSinkConfig openSearchSinkConfig) { private boolean basicAuthChanged(final ConnectionConfiguration newConfig) { final String existingUsername; final String existingPassword; - existingUsername = currentConfig.getUsername(); - existingPassword = currentConfig.getPassword(); + if (currentConfig.getAuthConfig() != null) { + existingUsername = currentConfig.getAuthConfig().getUsername(); + existingPassword = currentConfig.getAuthConfig().getPassword(); + } else { + existingUsername = currentConfig.getUsername(); + existingPassword = currentConfig.getPassword(); + } final String newUsername; final String newPassword; - newUsername = newConfig.getUsername(); - newPassword = newConfig.getPassword(); + if (newConfig.getAuthConfig() != null) { + newUsername = newConfig.getAuthConfig().getUsername(); + newPassword = newConfig.getAuthConfig().getPassword(); + } else { + newUsername = newConfig.getUsername(); + newPassword = newConfig.getPassword(); + } return !Objects.equals(existingUsername, newUsername) || !Objects.equals(existingPassword, newPassword); diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AuthConfig.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AuthConfig.java new file mode 100644 index 0000000000..e41f99043c --- /dev/null +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AuthConfig.java @@ -0,0 +1,19 @@ +package org.opensearch.dataprepper.plugins.sink.opensearch.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class AuthConfig { + @JsonProperty("username") + private String username; + + @JsonProperty("password") + private String password; + + public String getUsername() { + return username; + } + + public String getPassword() { + return password; + } +} diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java index d30c448db9..2bde73c184 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java @@ -32,6 +32,15 @@ public class OpenSearchSinkConfig { @JsonProperty("username") private String username = null; + @Getter + @JsonProperty("authentication") + private AuthConfig authConfig; + + @AssertTrue(message = "username and password should not be set when authentication is configured.") + public boolean isAuthConfigValid() { + return authConfig == null || (username == null && password == null); + } + @Getter @JsonProperty("password") private String password = null; diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java index a2356ccb94..9a4cf44b3e 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java @@ -180,6 +180,34 @@ void testReadConnectionConfigurationWithDeprecatedBasicCredentialsAndNoCert() th assertFalse(connectionConfiguration.isAwsSigv4()); } + @Test + void testReadConnectionConfigurationWithBasicCredentialsAndNoCert() throws JsonProcessingException { + final Map configurationMetadata = generateConfigurationMetadata( + TEST_HOSTS, null, null, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, null, false); + configurationMetadata.put("authentication", Map.of("username", TEST_USERNAME, "password", TEST_PASSWORD)); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(configurationMetadata); + final ConnectionConfiguration connectionConfiguration = + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); + assertEquals(TEST_HOSTS, connectionConfiguration.getHosts()); + assertNull(connectionConfiguration.getUsername()); + assertNull(connectionConfiguration.getPassword()); + assertNotNull(connectionConfiguration.getAuthConfig()); + assertEquals(TEST_USERNAME, connectionConfiguration.getAuthConfig().getUsername()); + assertEquals(TEST_PASSWORD, connectionConfiguration.getAuthConfig().getPassword()); + assertEquals(TEST_CONNECT_TIMEOUT, connectionConfiguration.getConnectTimeout()); + assertEquals(TEST_SOCKET_TIMEOUT, connectionConfiguration.getSocketTimeout()); + assertFalse(connectionConfiguration.isAwsSigv4()); + } + + @Test + void testReadConnectionConfigurationWithBothDeprecatedBasicCredentialsAndAuthConfigShouldThrow() throws JsonProcessingException { + final Map configurationMetadata = generateConfigurationMetadata( + TEST_HOSTS, TEST_USERNAME, null, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, null, false); + configurationMetadata.put("authentication", Map.of("username", TEST_USERNAME, "password", TEST_PASSWORD)); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(configurationMetadata); + assertFalse(openSearchSinkConfig.isAuthConfigValid()); + } + @Test void testCreateClientWithDeprecatedBasicCredentialsAndNoCert() throws IOException { final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresherTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresherTest.java index e31c680e8d..0ceb793785 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresherTest.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresherTest.java @@ -9,6 +9,7 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.client.opensearch.OpenSearchClient; import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.AuthConfig; import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig; import java.util.function.Function; @@ -40,6 +41,9 @@ class OpenSearchClientRefresherTest { @Mock private OpenSearchClient openSearchClient; + @Mock + private AuthConfig authConfig; + @Mock private PluginMetrics pluginMetrics; @@ -65,7 +69,7 @@ void testGet() { } @Test - void testGetAfterUpdateWithBasicAuthUnchanged() { + void testGetAfterUpdateWithDeprecatedBasicAuthUnchanged() { final OpenSearchClientRefresher objectUnderTest = createObjectUnderTest(); assertThat(objectUnderTest.get(), equalTo(openSearchClient)); verify(clientFunction, times(1)).apply(any()); @@ -86,7 +90,31 @@ void testGetAfterUpdateWithBasicAuthUnchanged() { } @Test - void testGetAfterUpdateWithUsernameChanged() { + void testGetAfterUpdateWithBasicAuthUnchanged() { + final OpenSearchClientRefresher objectUnderTest = createObjectUnderTest(); + assertThat(objectUnderTest.get(), equalTo(openSearchClient)); + verify(clientFunction, times(1)).apply(any()); + when(connectionConfiguration.getAuthConfig()).thenReturn(authConfig); + when(authConfig.getUsername()).thenReturn(TEST_USERNAME); + when(authConfig.getPassword()).thenReturn(TEST_PASSWORD); + final OpenSearchSinkConfig newConfig = mock(OpenSearchSinkConfig.class); + final ConnectionConfiguration newConnectionConfiguration = mock(ConnectionConfiguration.class); + final AuthConfig newAuthConfig = mock(AuthConfig.class); + when(newConnectionConfiguration.getAuthConfig()).thenReturn(newAuthConfig); + when(newAuthConfig.getUsername()).thenReturn(TEST_USERNAME); + when(newAuthConfig.getPassword()).thenReturn(TEST_PASSWORD); + try (MockedStatic configurationMockedStatic = mockStatic( + ConnectionConfiguration.class)) { + configurationMockedStatic.when(() -> ConnectionConfiguration.readConnectionConfiguration(eq(newConfig))) + .thenReturn(newConnectionConfiguration); + objectUnderTest.update(newConfig); + } + assertThat(objectUnderTest.get(), equalTo(openSearchClient)); + verifyNoMoreInteractions(clientFunction); + } + + @Test + void testGetAfterUpdateWithDeprecatedUsernameChanged() { when(pluginMetrics.counter(CREDENTIALS_CHANGED)).thenReturn(credentialsChangeCounter); final OpenSearchClientRefresher objectUnderTest = createObjectUnderTest(); assertThat(objectUnderTest.get(), equalTo(openSearchClient)); @@ -109,9 +137,36 @@ void testGetAfterUpdateWithUsernameChanged() { verify(clientFunction, times(2)).apply(any()); } + @Test + void testGetAfterUpdateWithUsernameChanged() { + when(pluginMetrics.counter(CREDENTIALS_CHANGED)).thenReturn(credentialsChangeCounter); + final OpenSearchClientRefresher objectUnderTest = createObjectUnderTest(); + assertThat(objectUnderTest.get(), equalTo(openSearchClient)); + verify(clientFunction, times(1)).apply(any()); + assertThat(objectUnderTest.get(), equalTo(openSearchClient)); + when(connectionConfiguration.getAuthConfig()).thenReturn(authConfig); + when(authConfig.getUsername()).thenReturn(TEST_USERNAME); + when(authConfig.getPassword()).thenReturn(TEST_PASSWORD); + final OpenSearchSinkConfig newConfig = mock(OpenSearchSinkConfig.class); + final ConnectionConfiguration newConnectionConfiguration = mock(ConnectionConfiguration.class); + final AuthConfig newAuthConfig = mock(AuthConfig.class); + when(newConnectionConfiguration.getAuthConfig()).thenReturn(newAuthConfig); + when(newAuthConfig.getUsername()).thenReturn(TEST_USERNAME + "_changed"); + final OpenSearchClient newClient = mock(OpenSearchClient.class); + when(clientFunction.apply(eq(newConnectionConfiguration))).thenReturn(newClient); + try (MockedStatic configurationMockedStatic = mockStatic( + ConnectionConfiguration.class)) { + configurationMockedStatic.when(() -> ConnectionConfiguration.readConnectionConfiguration(eq(newConfig))) + .thenReturn(newConnectionConfiguration); + objectUnderTest.update(newConfig); + } + assertThat(objectUnderTest.get(), equalTo(newClient)); + verify(credentialsChangeCounter).increment(); + verify(clientFunction, times(2)).apply(any()); + } @Test - void testGetAfterUpdateWithPasswordChanged() { + void testGetAfterUpdateWithDeprecatedPasswordChanged() { when(pluginMetrics.counter(CREDENTIALS_CHANGED)).thenReturn(credentialsChangeCounter); final OpenSearchClientRefresher objectUnderTest = createObjectUnderTest(); assertThat(objectUnderTest.get(), equalTo(openSearchClient)); @@ -136,6 +191,35 @@ void testGetAfterUpdateWithPasswordChanged() { verify(clientFunction, times(2)).apply(any()); } + @Test + void testGetAfterUpdateWithPasswordChanged() { + when(pluginMetrics.counter(CREDENTIALS_CHANGED)).thenReturn(credentialsChangeCounter); + final OpenSearchClientRefresher objectUnderTest = createObjectUnderTest(); + assertThat(objectUnderTest.get(), equalTo(openSearchClient)); + verify(clientFunction, times(1)).apply(any()); + assertThat(objectUnderTest.get(), equalTo(openSearchClient)); + when(connectionConfiguration.getAuthConfig()).thenReturn(authConfig); + when(authConfig.getUsername()).thenReturn(TEST_USERNAME); + when(authConfig.getPassword()).thenReturn(TEST_PASSWORD); + final OpenSearchSinkConfig newConfig = mock(OpenSearchSinkConfig.class); + final ConnectionConfiguration newConnectionConfiguration = mock(ConnectionConfiguration.class); + final AuthConfig newAuthConfig = mock(AuthConfig.class); + when(newConnectionConfiguration.getAuthConfig()).thenReturn(newAuthConfig); + when(newAuthConfig.getUsername()).thenReturn(TEST_USERNAME); + when(newAuthConfig.getPassword()).thenReturn(TEST_PASSWORD + "_changed"); + final OpenSearchClient newClient = mock(OpenSearchClient.class); + when(clientFunction.apply(eq(newConnectionConfiguration))).thenReturn(newClient); + try (MockedStatic configurationMockedStatic = mockStatic( + ConnectionConfiguration.class)) { + configurationMockedStatic.when(() -> ConnectionConfiguration.readConnectionConfiguration(eq(newConfig))) + .thenReturn(newConnectionConfiguration); + objectUnderTest.update(newConfig); + } + assertThat(objectUnderTest.get(), equalTo(newClient)); + verify(credentialsChangeCounter).increment(); + verify(clientFunction, times(2)).apply(any()); + } + @Test void testGetAfterUpdateClientFailure() { when(pluginMetrics.counter(CREDENTIALS_CHANGED)).thenReturn(credentialsChangeCounter); @@ -163,4 +247,4 @@ void testGetAfterUpdateClientFailure() { verify(clientRefreshErrorsCounter).increment(); verify(clientFunction, times(2)).apply(any()); } -} +} \ No newline at end of file