Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

schema revisions, add json aliases #5349

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ public class AggregateProcessorConfig {

@JsonPropertyDescription("An unordered list by which to group events. Events with the same values as these keys are put into the same group. " +
"If an event does not contain one of the <code>identification_keys</code>, then the value of that key is considered to be equal to <code>null</code>. " +
"At least one <code>identification_key</code> is required. An example configuration is [\"sourceIp\", \"destinationIp\", \"port\"].")
"At least one <code>identification_key</code> is required.")
@JsonProperty("identification_keys")
@NotEmpty
@ExampleValues({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ public class AnomalyDetectorProcessorConfig {
@UsesDataPrepperPlugin(pluginType = AnomalyDetectorMode.class)
private PluginModel detectorMode;

@JsonPropertyDescription("If provided, anomalies will be detected within each unique instance of these keys. For example, if you provide the ip field, anomalies will be detected separately for each unique IP address.")
@JsonPropertyDescription("If provided, anomalies will be detected within each unique instance of these keys. For example, if you provide the IP field, anomalies will be detected separately for each unique IP address.")
@JsonProperty("identification_keys")
@ExampleValues({
@Example(value = "ip_address", description = "Anomalies will be detected separately for each unique IP address from the existing ip_address key of the Event.")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ public class RandomCutForestModeConfig {
@JsonProperty(value = "time_decay", defaultValue = "" + DEFAULT_TIME_DECAY)
private double timeDecay = DEFAULT_TIME_DECAY;

@JsonPropertyDescription("Output after indicates the number of events to consume before outputting anomalies. Default is 32.")
@JsonPropertyDescription("The number of events to train the model before generating an anomaly event. Default is 32.")
@JsonProperty(value = "output_after", defaultValue = "" + DEFAULT_OUTPUT_AFTER)
private int outputAfter = DEFAULT_OUTPUT_AFTER;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
import org.opensearch.dataprepper.plugins.lambda.common.config.LambdaCommonConfig;

@JsonPropertyOrder
@JsonClassDescription("The <code>aws_lambda</code> processor enables invocation of an AWS Lambda function within your Data Prepper pipeline in order to process events." +
@JsonClassDescription("The <code>aws_lambda</code> processor enables invocation of an AWS Lambda function within your pipeline in order to process events. " +
"It supports both synchronous and asynchronous invocations based on your use case.")
public class LambdaProcessorConfig extends LambdaCommonConfig {
static final String DEFAULT_INVOCATION_TYPE = "request-response";
Expand All @@ -40,7 +40,7 @@ public class LambdaProcessorConfig extends LambdaCommonConfig {

@JsonPropertyDescription("Defines a condition for event to use this processor.")
@ExampleValues({
@Example(value = "event['status'] == 'process'", description = "The processor will only run on this condition.")
@Example(value = "/some_key == null", description = "The processor will only run on events where this condition evaluates to true.")
})
@JsonProperty("lambda_when")
private String whenCondition;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonPropertyDescription;
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
import com.fasterxml.jackson.annotation.JsonAlias;
import jakarta.validation.constraints.AssertTrue;
import org.opensearch.dataprepper.model.annotations.AlsoRequired;
import org.opensearch.dataprepper.model.annotations.ConditionalRequired;
Expand Down Expand Up @@ -133,8 +134,7 @@ public static boolean isValidPattern(final String pattern) {
private Boolean fromTimeReceived = DEFAULT_FROM_TIME_RECEIVED;

@JsonProperty("match")
@JsonPropertyDescription("The date match configuration. " +
"This option cannot be defined at the same time as <code>from_time_received</code>. " +
@JsonPropertyDescription("This option cannot be defined at the same time as <code>from_time_received</code>. " +
"The date processor will use the first pattern that matches each event's timestamp field. " +
"You must provide at least one pattern unless you have <code>from_time_received</code>.")
@AlsoRequired(values = {
Expand All @@ -155,9 +155,11 @@ public static boolean isValidPattern(final String pattern) {
})
private String outputFormat = DEFAULT_OUTPUT_FORMAT;

@JsonProperty("to_origination_metadata")
@JsonPropertyDescription("When <code>true</code>, the matched time is also added to the event's metadata as an instance of " +
"<code>Instant</code>. Default is <code>false</code>.")
@JsonProperty("origination_timestamp_to_metadata")
@JsonAlias("to_origination_metadata")
@JsonPropertyDescription("Include the origination timestamp in the metadata. " +
"Enabling this option will use this timestamp to report the EndToEndLatency metric " +
"when events reach the sink. Default is <code>false</code>.")
private Boolean toOriginationMetadata = DEFAULT_TO_ORIGINATION_METADATA;

@JsonProperty("source_timezone")
Expand Down Expand Up @@ -187,7 +189,7 @@ public static boolean isValidPattern(final String pattern) {
"or a string representation of the " +
"<a href=\"https://docs.oracle.com/javase/8/docs/api/java/util/Locale.html\">locale</a> object, such as <code>en_US</code>. " +
"A full list of locale fields, including language, country, and variant, can be found " +
"<a href=\"https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry\">here</a>." +
"<a href=\"https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry\">here</a>. " +
"Default is <code>Locale.ROOT</code>.")
@ExampleValues({
@Example("en-US"),
Expand All @@ -196,10 +198,10 @@ public static boolean isValidPattern(final String pattern) {
private String locale;

@JsonProperty("date_when")
@JsonPropertyDescription("Specifies under what condition the <code>date</code> processor should perform matching. " +
@JsonPropertyDescription("Specifies under what condition the <code>date</code> processor should run. " +
shenkw1 marked this conversation as resolved.
Show resolved Hide resolved
"Default is no condition.")
@ExampleValues({
@Example(value = "/some_key == null", description = "Only runs the date processor on the Event if some_key is null or doesn't exist.")
@Example(value = "/some_key == null", description = "The processor will only run when on events where this condition evaluates to true.")
shenkw1 marked this conversation as resolved.
Show resolved Hide resolved
})
private String dateWhen;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public class DropEventProcessorConfig {
})
private String dropWhen;

@JsonPropertyDescription("Specifies how exceptions are handled when an exception occurs while evaluating an event. Default value is <code>skip</code>, which drops the event so that it is not sent to further processors or sinks.")
@JsonPropertyDescription("Specifies how exceptions are handled when an exception occurs while evaluating an event. Default value is <code>skip</code>, which sends the events to further processors or sinks.")
shenkw1 marked this conversation as resolved.
Show resolved Hide resolved
@JsonProperty(value = "handle_failed_events", defaultValue = "skip")
private HandleFailedEventsOption handleFailedEventsOption = HandleFailedEventsOption.SKIP;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@

import com.fasterxml.jackson.annotation.JsonClassDescription;
import com.fasterxml.jackson.annotation.JsonProperty;

import com.fasterxml.jackson.annotation.JsonPropertyDescription;
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
import com.fasterxml.jackson.annotation.JsonAlias;
import jakarta.validation.constraints.AssertTrue;
import jakarta.validation.constraints.NotNull;
import org.opensearch.dataprepper.model.annotations.ExampleValues;
Expand All @@ -20,7 +20,7 @@
import java.util.List;

@JsonPropertyOrder
@JsonClassDescription("The <code>flatten</code> processor transforms nested objects inside of events into flattened structures.")
@JsonClassDescription("The <code>flatten</code> processor transforms nested objects inside of events into flattened structures.")
public class FlattenProcessorConfig {
static final String REMOVE_LIST_INDICES_KEY = "remove_list_indices";

Expand All @@ -44,12 +44,14 @@ public class FlattenProcessorConfig {
})
private String target;

@JsonProperty("remove_processed_fields")
@JsonProperty("remove_source_keys")
@JsonAlias("remove_processed_fields")
@JsonPropertyDescription("When <code>true</code>, the processor removes all processed fields from the source. " +
"The default is <code>false</code> which leaves the source fields.")
private boolean removeProcessedFields = false;

@JsonProperty(REMOVE_LIST_INDICES_KEY)
@JsonProperty("remove_list_elements")
@JsonAlias(REMOVE_LIST_INDICES_KEY)
@JsonPropertyDescription("When <code>true</code>, the processor converts the fields from the source map into lists and " +
"puts the lists into the target field. Default is <code>false</code>.")
private boolean removeListIndices = false;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -195,8 +195,7 @@ public class KeyValueProcessorConfig {
"If this flag is enabled, then the content between the delimiters is considered to be one entity and " +
"they are not parsed as key-value pairs. The following characters are used a group delimiters: " +
"<code>{...}</code>, <code>[...]</code>, <code>&lt;...&gt;</code>, <code>(...)</code>, <code>\"...\"</code>, <code>'...'</code>, <code>http://... (space)</code>, and <code>https:// (space)</code>. " +
"Default is <code>false</code>. For example, if <code>value_grouping</code> is <code>true</code>, then " +
"<code>{\"key1=[a=b,c=d]&amp;key2=value2\"}</code> parses to <code>{\"key1\": \"[a=b,c=d]\", \"key2\": \"value2\"}</code>.")
"Default is <code>false</code>.")
@AlsoRequired(values = {
@AlsoRequired.Required(name = FIELD_DELIMITER_REGEX_KEY, allowedValues = {"null"})
})
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ public class AddEntryProcessorConfig {
public static class Entry {
@JsonPropertyDescription("The key of the new entry to be added. Some examples of keys include <code>my_key</code>, " +
"<code>myKey</code>, and <code>object/sub_Key</code>. The key can also be a format expression, for example, <code>${/key1}</code> to " +
"use the value of field <code>key1</code> as the key. Either one of <code>key</code> or <code>metadata_key</code> is required.")
"use the value of field <code>key1</code> as the key. Exactly one of <code>key</code> or <code>metadata_key</code> is required.")
@AlsoRequired(values = {
@AlsoRequired.Required(name=METADATA_KEY_KEY, allowedValues = {"null"})
})
Expand All @@ -79,7 +79,8 @@ public static class Entry {

@JsonProperty(METADATA_KEY_KEY)
@JsonPropertyDescription("The key for the new metadata attribute. The argument must be a literal string key " +
"and not a JSON Pointer. Either one of <code>key</code> or <code>metadata_key</code> is required.")
"and not a JSON Pointer. Adds an attribute to the Events that will not be sent to the sinks, but can be used for condition expressions and routing with the getMetadata function. " +
"Exactly one of <code>key</code> or <code>metadata_key</code> is required.")
@AlsoRequired(values = {
@AlsoRequired.Required(name="key", allowedValues = {"null"})
})
Expand All @@ -103,7 +104,7 @@ public static class Entry {
private Object value;

@JsonPropertyDescription("A format string to use as the value of the new entry, for example, " +
"<code>${key1}-${key2}</code>, where <code>key1</code> and <code>key2</code> are existing keys in the event. Required if neither" +
"<code>${key1}-${key2}</code>, where <code>key1</code> and <code>key2</code> are existing keys in the event. Required if neither " +
"<code>value</code> nor <code>value_expression</code> is specified.")
@AlsoRequired(values = {
@AlsoRequired.Required(name="value", allowedValues = {"null"}),
Expand Down Expand Up @@ -132,7 +133,7 @@ public static class Entry {

@JsonProperty(OVERWRITE_IF_KEY_EXISTS_KEY)
@JsonPropertyDescription("When set to <code>true</code>, the existing value is overwritten if <code>key</code> already exists " +
"in the event. The default value is <code>false</code>.")
"in the event. Only one of <code>overwrite_if_key_exists</code> or <code>append_if_key_exists</code> can be <code>true</code>. The default value is <code>false</code>.")
@AlsoRequired(values = {
@AlsoRequired.Required(name=APPEND_IF_KEY_EXISTS_KEY, allowedValues = {"false"})
})
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
import com.fasterxml.jackson.annotation.JsonPropertyDescription;
import com.fasterxml.jackson.annotation.JsonAlias;
import org.opensearch.dataprepper.model.annotations.AlsoRequired;
import org.opensearch.dataprepper.model.annotations.ConditionalRequired;
import org.opensearch.dataprepper.model.annotations.ConditionalRequired.IfThenElse;
Expand Down Expand Up @@ -55,7 +56,8 @@ public class ConvertEntryTypeProcessorConfig implements ConverterArguments {
@JsonPropertyDescription("Target type for the values. Default value is <code>integer</code>.")
private TargetType type = TargetType.INTEGER;

@JsonProperty("null_values")
@JsonProperty("null_conversion_values")
@JsonAlias("null_values")
@JsonPropertyDescription("String representation of what constitutes a null value. If the field value equals one of these strings, then the value is considered null and is converted to null.")
private List<String> nullValues;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
import java.util.List;

@JsonPropertyOrder
@JsonClassDescription("The <code>copy_values</code> processor copies values within an event to other fields within the event.")
@JsonClassDescription("The <code>copy_values</code> processor copies values from an event to another key in an event.")
public class CopyValueProcessorConfig {
static final String FROM_LIST_KEY = "from_list";
static final String TO_LIST_KEY = "to_list";
Expand All @@ -30,15 +30,13 @@ public static class Entry {
@NotEmpty
@NotNull
@JsonProperty("from_key")
@JsonPropertyDescription("The key of the entry to be copied. Either <code>from_key</code> and " +
"<code>to_key</code> or <code>from_list</code> and <code>to_list</code> must be defined.")
@JsonPropertyDescription("The key of the entry to be copied. This must be configured.")
private String fromKey;

@NotEmpty
@NotNull
@JsonProperty("to_key")
@JsonPropertyDescription("The key of the new entry to be added. Either <code>from_key</code> and " +
"<code>to_key</code> or <code>from_list</code> and <code>to_list</code> must be defined.")
@JsonPropertyDescription("The key of the new entry to be added. This must be configured.")
private String toKey;

@JsonProperty("overwrite_if_to_key_exists")
Expand Down Expand Up @@ -88,16 +86,16 @@ public Entry() {
private List<Entry> entries;

@JsonProperty(FROM_LIST_KEY)
@JsonPropertyDescription("The key of the list of objects to be copied. Either <code>from_key</code> and " +
"<code>to_key</code> or <code>from_list</code> and <code>to_list</code> must be defined.")
@JsonPropertyDescription("The key of the list of objects to be copied. " +
"Both <code>from_key</code> and <code>to_key</code> must be configured and will be applied on the corresponding list.")
@AlsoRequired(values = {
@AlsoRequired.Required(name = TO_LIST_KEY)
})
private String fromList;

@JsonProperty(TO_LIST_KEY)
@JsonPropertyDescription("The key of the new list to be added. Either <code>from_key</code> and " +
"<code>to_key</code> or <code>from_list</code> and <code>to_list</code> must be defined.")
@JsonPropertyDescription("The key of the new list to be added. " +
"Both <code>from_key</code> and <code>to_key</code> must be configured and will be applied on the corresponding list.")
@AlsoRequired(values = {
@AlsoRequired.Required(name = FROM_LIST_KEY)
})
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
public class ObfuscationProcessorConfig {

@JsonProperty("source")
@JsonPropertyDescription("The source field to obfuscate.")
@JsonPropertyDescription("The source key to obfuscate. Default action is to mask with <code>*</code>.")
@NotEmpty
@NotNull
@ExampleValues({
Expand Down
Loading