Merge main-opsathon to main

plugins/inputs/logfile/README.md

@@ -47,6 +47,7 @@ The plugin expects messages in one of the
       timestamp_regex = "^(\\d{2} \\w{3} \\d{4} \\d{2}:\\d{2}:\\d{2}).*$"
       timestamp_layout = ["_2 Jan 2006 15:04:05"]
       timezone = "UTC"
+      trim_timestamp = false
       multi_line_start_pattern = "{timestamp_regex}"
       ## Read file from beginning.
       from_beginning = false
@@ -65,6 +66,7 @@ The plugin expects messages in one of the
       timestamp_regex = "^(\\d{2} \\w{3} \\d{4} \\d{2}:\\d{2}:\\d{2}).*$"
       timestamp_layout = ["_2 Jan 2006 15:04:05"]
       timezone = "UTC"
+      trim_timestamp = true
       multi_line_start_pattern = "{timestamp_regex}"
       ## Read file from beginning.
       from_beginning = false

plugins/inputs/logfile/fileconfig.go

@@ -48,6 +48,8 @@ type FileConfig struct {
 	TimestampLayout []string `toml:"timestamp_layout"`
 	//The time zone used to parse the timestampFromLogLine in the log entry.
 	Timezone string `toml:"timezone"`
+	//Trim timestamp from log line
+	TrimTimestamp bool `toml:"trim_timestamp"`
 
 	//Indicate whether it is a start of multiline.
 	//If this config is not present, it means the multiline mode is disabled.
@@ -171,9 +173,9 @@ func (config *FileConfig) init() error {
 // Try to parse the timestampFromLogLine value from the log entry line.
 // The parser logic will be based on the timestampFromLogLine regex, and time zone info.
 // If the parsing operation encounters any issue, int64(0) is returned.
-func (config *FileConfig) timestampFromLogLine(logValue string) time.Time {
+func (config *FileConfig) timestampFromLogLine(logValue string) (time.Time, string) {
 	if config.TimestampRegexP == nil {
-		return time.Time{}
+		return time.Time{}, logValue
 	}
 	index := config.TimestampRegexP.FindStringSubmatchIndex(logValue)
 	if len(index) > 3 {
@@ -196,7 +198,7 @@ func (config *FileConfig) timestampFromLogLine(logValue string) time.Time {
 		}
 		if err != nil {
 			log.Printf("E! Error parsing timestampFromLogLine: %s", err)
-			return time.Time{}
+			return time.Time{}, logValue
 		}
 		if timestamp.Year() == 0 {
 			now := time.Now()
@@ -208,9 +210,14 @@ func (config *FileConfig) timestampFromLogLine(logValue string) time.Time {
 				timestamp = timestamp.AddDate(-1, 0, 0)
 			}
 		}
-		return timestamp
+		if config.TrimTimestamp {
+			// Trim the entire timestamp portion and leading whitespaces
+			// The whitespace characters being removed are: space, tab, newline, and carriage return ( " \t\n\r")
+			return timestamp, strings.TrimLeft(logValue[:index[0]]+logValue[index[1]:], " \t\n\r")
+		}
+		return timestamp, logValue
 	}
-	return time.Time{}
+	return time.Time{}, logValue
 }
 
 // This method determine whether the line is a start line for multiline log entry.

plugins/inputs/logfile/fileconfig_test.go

@@ -130,15 +130,34 @@ func TestTimestampParser(t *testing.T) {
 	expectedTimestamp := time.Unix(1497882318, 0)
 	timestampString := "19 Jun 2017 14:25:18"
 	logEntry := fmt.Sprintf("%s [INFO] This is a test message.", timestampString)
-	timestamp := fileConfig.timestampFromLogLine(logEntry)
+	timestamp, modifiedLogEntry := fileConfig.timestampFromLogLine(logEntry)
 	assert.Equal(t, expectedTimestamp.UnixNano(), timestamp.UnixNano(),
 		fmt.Sprintf("The timestampFromLogLine value %v is not the same as expected %v.", timestamp, expectedTimestamp))
+	assert.Equal(t, logEntry, modifiedLogEntry)
 
 	// Test regex match for multiline, the first timestamp in multiline should be matched
 	logEntry = fmt.Sprintf("%s [INFO] This is the first line.\n19 Jun 2017 14:25:19 [INFO] This is the second line.\n", timestampString)
-	timestamp = fileConfig.timestampFromLogLine(logEntry)
+	timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry)
 	assert.Equal(t, expectedTimestamp.UnixNano(), timestamp.UnixNano(),
 		fmt.Sprintf("The timestampFromLogLine value %v is not the same as expected %v.", timestamp, expectedTimestamp))
+	assert.Equal(t, logEntry, modifiedLogEntry)
+
+	// Test TrimTimeStamp for single line
+	fileConfig.TrimTimestamp = true
+	logEntry = fmt.Sprintf("%s [INFO] This is a test message.", timestampString)
+	trimmedTimestampString := "[INFO] This is a test message."
+	timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry)
+	assert.Equal(t, expectedTimestamp.UnixNano(), timestamp.UnixNano(),
+		fmt.Sprintf("The timestampFromLogLine value %v is not the same as expected %v.", timestamp, expectedTimestamp))
+	assert.Equal(t, trimmedTimestampString, modifiedLogEntry)
+
+	// Test TrimTimeStamp for multiline, the first timestamp in multiline should be matched
+	logEntry = fmt.Sprintf("%s [INFO] This is the first line.\n19 Jun 2017 14:25:19 [INFO] This is the second line.\n", timestampString)
+	trimmedTimestampString = "[INFO] This is the first line.\n19 Jun 2017 14:25:19 [INFO] This is the second line.\n"
+	timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry)
+	assert.Equal(t, expectedTimestamp.UnixNano(), timestamp.UnixNano(),
+		fmt.Sprintf("The timestampFromLogLine value %v is not the same as expected %v.", timestamp, expectedTimestamp))
+	assert.Equal(t, trimmedTimestampString, modifiedLogEntry)
 }
 
 func TestTimestampParserWithPadding(t *testing.T) {
@@ -155,15 +174,39 @@ func TestTimestampParserWithPadding(t *testing.T) {
 		Timezone:        timezone,
 		TimezoneLoc:     timezoneLoc}
 
-	logEntry := fmt.Sprintf(" 2 1 07:10:06 instance-id: i-02fce21a425a2efb3")
-	timestamp := fileConfig.timestampFromLogLine(logEntry)
+	logEntry := " 2 1 07:10:06 instance-id: i-02fce21a425a2efb3"
+	timestamp, modifiedLogEntry := fileConfig.timestampFromLogLine(logEntry)
 	assert.Equal(t, 7, timestamp.Hour(), fmt.Sprintf("Timestamp does not match: %v, act: %v", "7", timestamp.Hour()))
 	assert.Equal(t, 10, timestamp.Minute(), fmt.Sprintf("Timestamp does not match: %v, act: %v", "10", timestamp.Minute()))
+	assert.Equal(t, logEntry, modifiedLogEntry)
 
-	logEntry = fmt.Sprintf("2 1 07:10:06 instance-id: i-02fce21a425a2efb3")
-	timestamp = fileConfig.timestampFromLogLine(logEntry)
+	logEntry = "2 1 07:10:06 instance-id: i-02fce21a425a2efb3"
+	timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry)
 	assert.Equal(t, 7, timestamp.Hour(), fmt.Sprintf("Timestamp does not match: %v, act: %v", "7", timestamp.Hour()))
 	assert.Equal(t, 10, timestamp.Minute(), fmt.Sprintf("Timestamp does not match: %v, act: %v", "10", timestamp.Minute()))
+	assert.Equal(t, logEntry, modifiedLogEntry)
+
+	//Test when TrimTimeStamp is enabled
+	fileConfig.TrimTimestamp = true
+	logEntry = " 2 1 07:10:06 instance-id: i-02fce21a425a2efb3"
+	trimmedTimestampString := "instance-id: i-02fce21a425a2efb3"
+	timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry)
+	assert.Equal(t, 7, timestamp.Hour(), fmt.Sprintf("Timestamp does not match: %v, act: %v", "7", timestamp.Hour()))
+	assert.Equal(t, 10, timestamp.Minute(), fmt.Sprintf("Timestamp does not match: %v, act: %v", "10", timestamp.Minute()))
+	assert.Equal(t, trimmedTimestampString, modifiedLogEntry)
+
+	logEntry = "2 1 07:10:06 instance-id: i-02fce21a425a2efb3"
+	timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry)
+	assert.Equal(t, 7, timestamp.Hour(), fmt.Sprintf("Timestamp does not match: %v, act: %v", "7", timestamp.Hour()))
+	assert.Equal(t, 10, timestamp.Minute(), fmt.Sprintf("Timestamp does not match: %v, act: %v", "10", timestamp.Minute()))
+	assert.Equal(t, trimmedTimestampString, modifiedLogEntry)
+
+	logEntry = " instance-id: i-02fce21a425a2efb3 2 1 07:10:06"
+	trimmedTimestampString = "instance-id: i-02fce21a425a2efb3 "
+	timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry)
+	assert.Equal(t, 7, timestamp.Hour(), fmt.Sprintf("Timestamp does not match: %v, act: %v", "7", timestamp.Hour()))
+	assert.Equal(t, 10, timestamp.Minute(), fmt.Sprintf("Timestamp does not match: %v, act: %v", "10", timestamp.Minute()))
+	assert.Equal(t, trimmedTimestampString, modifiedLogEntry)
 }
 
 func TestTimestampParserDefault(t *testing.T) {
@@ -183,26 +226,56 @@ func TestTimestampParserDefault(t *testing.T) {
 		TimezoneLoc:     timezoneLoc}
 
 	// make sure layout is compatible for "Sep 9", "Sep  9" , "Sep 09", "Sep  09" options
-	logEntry := fmt.Sprintf("Sep 9 02:00:43  ip-10-4-213-132 \n")
-	timestamp := fileConfig.timestampFromLogLine(logEntry)
+	logEntry := "Sep 9 02:00:43  ip-10-4-213-132 \n"
+	timestamp, modifiedLogEntry := fileConfig.timestampFromLogLine(logEntry)
+	assert.Equal(t, 02, timestamp.Hour())
+	assert.Equal(t, 00, timestamp.Minute())
+	assert.Equal(t, logEntry, modifiedLogEntry)
+
+	logEntry = "Sep  9 02:00:43  ip-10-4-213-132 \n"
+	timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry)
+	assert.Equal(t, 02, timestamp.Hour())
+	assert.Equal(t, 00, timestamp.Minute())
+	assert.Equal(t, logEntry, modifiedLogEntry)
+
+	logEntry = "Sep 09 02:00:43  ip-10-4-213-132 \n"
+	timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry)
 	assert.Equal(t, 02, timestamp.Hour())
 	assert.Equal(t, 00, timestamp.Minute())
+	assert.Equal(t, logEntry, modifiedLogEntry)
 
-	logEntry = fmt.Sprintf("Sep  9 02:00:43  ip-10-4-213-132 \n")
-	timestamp = fileConfig.timestampFromLogLine(logEntry)
+	logEntry = "Sep  09 02:00:43  ip-10-4-213-132 \n"
+	timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry)
 	assert.Equal(t, 02, timestamp.Hour())
 	assert.Equal(t, 00, timestamp.Minute())
+	assert.Equal(t, logEntry, modifiedLogEntry)
 
-	logEntry = fmt.Sprintf("Sep 09 02:00:43  ip-10-4-213-132 \n")
-	timestamp = fileConfig.timestampFromLogLine(logEntry)
+	// When TrimTimestamp is enabled, make sure layout is compatible for "Sep 9", "Sep  9" , "Sep 09", "Sep  09" options and log value is trimmed correctly
+	fileConfig.TrimTimestamp = true
+	logEntry = "Sep 9 02:00:43  ip-10-4-213-132 \n"
+	trimmedTimestampString := "ip-10-4-213-132 \n"
+	timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry)
 	assert.Equal(t, 02, timestamp.Hour())
 	assert.Equal(t, 00, timestamp.Minute())
+	assert.Equal(t, trimmedTimestampString, modifiedLogEntry)
 
-	logEntry = fmt.Sprintf("Sep  09 02:00:43  ip-10-4-213-132 \n")
-	timestamp = fileConfig.timestampFromLogLine(logEntry)
+	logEntry = "Sep  9 02:00:43  ip-10-4-213-132 \n"
+	timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry)
 	assert.Equal(t, 02, timestamp.Hour())
 	assert.Equal(t, 00, timestamp.Minute())
+	assert.Equal(t, trimmedTimestampString, modifiedLogEntry)
 
+	logEntry = "Sep 09 02:00:43  ip-10-4-213-132 \n"
+	timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry)
+	assert.Equal(t, 02, timestamp.Hour())
+	assert.Equal(t, 00, timestamp.Minute())
+	assert.Equal(t, trimmedTimestampString, modifiedLogEntry)
+
+	logEntry = "Sep  09 02:00:43  ip-10-4-213-132 \n"
+	timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry)
+	assert.Equal(t, 02, timestamp.Hour())
+	assert.Equal(t, 00, timestamp.Minute())
+	assert.Equal(t, trimmedTimestampString, modifiedLogEntry)
 }
 
 func TestTimestampParserWithFracSeconds(t *testing.T) {
@@ -222,15 +295,34 @@ func TestTimestampParserWithFracSeconds(t *testing.T) {
 	expectedTimestamp := time.Unix(1497882318, 234000000)
 	timestampString := "19 Jun 2017 14:25:18,234088 UTC"
 	logEntry := fmt.Sprintf("%s [INFO] This is a test message.", timestampString)
-	timestamp := fileConfig.timestampFromLogLine(logEntry)
+	timestamp, modifiedLogEntry := fileConfig.timestampFromLogLine(logEntry)
 	assert.Equal(t, expectedTimestamp.UnixNano(), timestamp.UnixNano(),
 		fmt.Sprintf("The timestampFromLogLine value %v is not the same as expected %v.", timestamp, expectedTimestamp))
+	assert.Equal(t, logEntry, modifiedLogEntry)
 
 	// Test regex match for multiline, the first timestamp in multiline should be matched
 	logEntry = fmt.Sprintf("%s [INFO] This is the first line.\n19 Jun 2017 14:25:19,123456 UTC [INFO] This is the second line.\n", timestampString)
-	timestamp = fileConfig.timestampFromLogLine(logEntry)
+	timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry)
+	assert.Equal(t, expectedTimestamp.UnixNano(), timestamp.UnixNano(),
+		fmt.Sprintf("The timestampFromLogLine value %v is not the same as expected %v.", timestamp, expectedTimestamp))
+	assert.Equal(t, logEntry, modifiedLogEntry)
+
+	// Test TrimTimeStamp for single line
+	fileConfig.TrimTimestamp = true
+	logEntry = fmt.Sprintf("%s [INFO] This is a test message.", timestampString)
+	trimmedTimestampString := "[INFO] This is a test message."
+	timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry)
+	assert.Equal(t, expectedTimestamp.UnixNano(), timestamp.UnixNano(),
+		fmt.Sprintf("The timestampFromLogLine value %v is not the same as expected %v.", timestamp, expectedTimestamp))
+	assert.Equal(t, trimmedTimestampString, modifiedLogEntry)
+
+	// Test TrimTimeStamp for multiline, the first timestamp in multiline should be matched
+	logEntry = fmt.Sprintf("%s [INFO] This is the first line.\n19 Jun 2017 14:25:19,123456 UTC [INFO] This is the second line.\n", timestampString)
+	trimmedTimestampString = "[INFO] This is the first line.\n19 Jun 2017 14:25:19,123456 UTC [INFO] This is the second line.\n"
+	timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry)
 	assert.Equal(t, expectedTimestamp.UnixNano(), timestamp.UnixNano(),
 		fmt.Sprintf("The timestampFromLogLine value %v is not the same as expected %v.", timestamp, expectedTimestamp))
+	assert.Equal(t, trimmedTimestampString, modifiedLogEntry)
 }
 
 func TestNonAllowlistedTimezone(t *testing.T) {

plugins/inputs/logfile/logfile.go

@@ -76,6 +76,7 @@ const sampleConfig = `
       timestamp_regex = "^(\\d{2} \\w{3} \\d{4} \\d{2}:\\d{2}:\\d{2}).*$"
       timestamp_layout = ["_2 Jan 2006 15:04:05"]
       timezone = "UTC"
+      trim_timestamp = false
       multi_line_start_pattern = "{timestamp_regex}"
       ## Read file from beginning.
       from_beginning = false

plugins/inputs/logfile/tailersrc.go

@@ -67,7 +67,7 @@ type tailerSrc struct {
 	stateFilePath   string
 	tailer          *tail.Tail
 	autoRemoval     bool
-	timestampFn     func(string) time.Time
+	timestampFn     func(string) (time.Time, string)
 	enc             encoding.Encoding
 	maxEventSize    int
 	truncateSuffix  string
@@ -91,7 +91,7 @@ func NewTailerSrc(
 	autoRemoval bool,
 	isMultilineStartFn func(string) bool,
 	filters []*LogFilter,
-	timestampFn func(string) time.Time,
+	timestampFn func(string) (time.Time, string),
 	enc encoding.Encoding,
 	maxEventSize int,
 	truncateSuffix string,
@@ -195,9 +195,10 @@ func (ts *tailerSrc) runTail() {
 			if !ok {
 				if msgBuf.Len() > 0 {
 					msg := msgBuf.String()
+					timestamp, modifiedMsg := ts.timestampFn(msg)
 					e := &LogEvent{
-						msg:    msg,
-						t:      ts.timestampFn(msg),
+						msg:    modifiedMsg,
+						t:      timestamp,
 						offset: *fo,
 						src:    ts,
 					}
@@ -249,9 +250,10 @@ func (ts *tailerSrc) runTail() {
 
 			if msgBuf.Len() > 0 {
 				msg := msgBuf.String()
+				timestamp, modifiedMsg := ts.timestampFn(msg)
 				e := &LogEvent{
-					msg:    msg,
-					t:      ts.timestampFn(msg),
+					msg:    modifiedMsg,
+					t:      timestamp,
 					offset: *fo,
 					src:    ts,
 				}
@@ -276,9 +278,10 @@ func (ts *tailerSrc) runTail() {
 			}
 
 			msg := msgBuf.String()
+			timestamp, modifiedMsg := ts.timestampFn(msg)
 			e := &LogEvent{
-				msg:    msg,
-				t:      ts.timestampFn(msg),
+				msg:    modifiedMsg,
+				t:      timestamp,
 				offset: *fo,
 				src:    ts,
 			}

plugins/inputs/logfile/tailersrc_test.go

@@ -324,15 +324,15 @@ func TestTailerSrcFiltersMultiLineLogs(t *testing.T) {
 	assertExpectedLogsPublished(t, n, int(*resources.consumed))
 }
 
-func parseRFC3339Timestamp(line string) time.Time {
+func parseRFC3339Timestamp(line string) (time.Time, string) {
 	// Use RFC3339 for testing `2006-01-02T15:04:05Z07:00`
 	re := regexp.MustCompile(`\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[Z+\-]\d{2}:\d{2}`)
 	tstr := re.FindString(line)
 	var t time.Time
 	if tstr != "" {
 		t, _ = time.Parse(time.RFC3339, tstr)
 	}
-	return t
+	return t, line
 }
 
 func logLine(s string, l int, t time.Time) string {

translator/config/schema.json

@@ -959,6 +959,10 @@
                       "UTC"
                     ]
                   },
+                  "trim_timestamp" : {
+                    "type": "boolean",
+                    "description": "Whether to trim the timestamp in the log message"
+                  },
                   "encoding": {
                     "type": "string",
                     "minLength": 1,

translator/tocwconfig/sampleConfig/complete_linux_config.conf

@@ -54,6 +54,7 @@
       pipe = false
       retention_in_days = 5
       timezone = "UTC"
+      trim_timestamp = true
 
     [[inputs.logfile.file_config]]
       auto_removal = true

translator/tocwconfig/sampleConfig/complete_linux_config.json

@@ -258,6 +258,7 @@
             "log_group_name": "amazon-cloudwatch-agent.log",
             "log_stream_name": "amazon-cloudwatch-agent.log",
             "timezone": "UTC",
+            "trim_timestamp": true,
             "retention_in_days": 5
           },
           {

translator/tocwconfig/totomlconfig/testdata/agentToml.conf

@@ -54,6 +54,7 @@
       pipe = false
       retention_in_days = 5
       timezone = "UTC"
+      trim_timestamp = true
 
     [[inputs.logfile.file_config]]
       auto_removal = true

translator/tocwconfig/totomlconfig/testdata/agentToml.json

@@ -182,7 +182,8 @@
             "log_group_name": "amazon-cloudwatch-agent.log",
             "log_stream_name": "amazon-cloudwatch-agent.log",
             "timezone": "UTC",
-            "retention_in_days": 5
+            "retention_in_days": 5,
+            "trim_timestamp": true
           },
           {
             "file_path": "/opt/aws/amazon-cloudwatch-agent/logs/test.log",

translator/tocwconfig/totomlconfig/tomlConfigTemplate/tomlConfig.go

@@ -126,6 +126,8 @@ type (
 		Pipe            bool
 		RetentionInDays int `toml:"retention_in_days"`
 		Timezone        string
+		//Customer specifies if the timestamp from the log message should be trimmed
+		TrimTimestamp bool `toml:"trim_timestamp"`
 		//Customer specified service.name
 		ServiceName string `toml:"service_name"`
 		//Customer specified deployment.environment