From 26c1a5259d12abbbdb7cb32ee3fd5893de182699 Mon Sep 17 00:00:00 2001 From: POOJA REDDY NATHALA Date: Wed, 26 Feb 2025 23:20:26 +0530 Subject: [PATCH 1/2] Trims timestamp from log message if customer enables on logs plugin (#1568) --- plugins/inputs/logfile/README.md | 2 + plugins/inputs/logfile/fileconfig.go | 16 ++- plugins/inputs/logfile/fileconfig_test.go | 118 +++++++++++++++--- plugins/inputs/logfile/logfile.go | 1 + plugins/inputs/logfile/tailersrc.go | 19 +-- plugins/inputs/logfile/tailersrc_test.go | 4 +- translator/config/schema.json | 4 + .../sampleConfig/complete_linux_config.conf | 1 + .../sampleConfig/complete_linux_config.json | 1 + .../totomlconfig/testdata/agentToml.conf | 1 + .../totomlconfig/testdata/agentToml.json | 3 +- .../tomlConfigTemplate/tomlConfig.go | 2 + .../files/collect_list/ruleTrimTimestamp.go | 33 +++++ 13 files changed, 173 insertions(+), 32 deletions(-) create mode 100644 translator/translate/logs/logs_collected/files/collect_list/ruleTrimTimestamp.go diff --git a/plugins/inputs/logfile/README.md b/plugins/inputs/logfile/README.md index e514ce75c4..8e2af5f67c 100644 --- a/plugins/inputs/logfile/README.md +++ b/plugins/inputs/logfile/README.md @@ -47,6 +47,7 @@ The plugin expects messages in one of the timestamp_regex = "^(\\d{2} \\w{3} \\d{4} \\d{2}:\\d{2}:\\d{2}).*$" timestamp_layout = ["_2 Jan 2006 15:04:05"] timezone = "UTC" + trim_timestamp = false multi_line_start_pattern = "{timestamp_regex}" ## Read file from beginning. from_beginning = false @@ -65,6 +66,7 @@ The plugin expects messages in one of the timestamp_regex = "^(\\d{2} \\w{3} \\d{4} \\d{2}:\\d{2}:\\d{2}).*$" timestamp_layout = ["_2 Jan 2006 15:04:05"] timezone = "UTC" + trim_timestamp = true multi_line_start_pattern = "{timestamp_regex}" ## Read file from beginning. from_beginning = false diff --git a/plugins/inputs/logfile/fileconfig.go b/plugins/inputs/logfile/fileconfig.go index 1f41abe033..23a17cc96c 100644 --- a/plugins/inputs/logfile/fileconfig.go +++ b/plugins/inputs/logfile/fileconfig.go @@ -48,6 +48,8 @@ type FileConfig struct { TimestampLayout []string `toml:"timestamp_layout"` //The time zone used to parse the timestampFromLogLine in the log entry. Timezone string `toml:"timezone"` + //Trim timestamp from log line + TrimTimestamp bool `toml:"trim_timestamp"` //Indicate whether it is a start of multiline. //If this config is not present, it means the multiline mode is disabled. @@ -171,9 +173,9 @@ func (config *FileConfig) init() error { // Try to parse the timestampFromLogLine value from the log entry line. // The parser logic will be based on the timestampFromLogLine regex, and time zone info. // If the parsing operation encounters any issue, int64(0) is returned. -func (config *FileConfig) timestampFromLogLine(logValue string) time.Time { +func (config *FileConfig) timestampFromLogLine(logValue string) (time.Time, string) { if config.TimestampRegexP == nil { - return time.Time{} + return time.Time{}, logValue } index := config.TimestampRegexP.FindStringSubmatchIndex(logValue) if len(index) > 3 { @@ -196,7 +198,7 @@ func (config *FileConfig) timestampFromLogLine(logValue string) time.Time { } if err != nil { log.Printf("E! Error parsing timestampFromLogLine: %s", err) - return time.Time{} + return time.Time{}, logValue } if timestamp.Year() == 0 { now := time.Now() @@ -208,9 +210,13 @@ func (config *FileConfig) timestampFromLogLine(logValue string) time.Time { timestamp = timestamp.AddDate(-1, 0, 0) } } - return timestamp + if config.TrimTimestamp { + // Trim the entire timestamp portion (from start to end of the match) + return timestamp, logValue[:index[0]] + logValue[index[1]:] + } + return timestamp, logValue } - return time.Time{} + return time.Time{}, logValue } // This method determine whether the line is a start line for multiline log entry. diff --git a/plugins/inputs/logfile/fileconfig_test.go b/plugins/inputs/logfile/fileconfig_test.go index 7baa7cbbea..f0c974e266 100644 --- a/plugins/inputs/logfile/fileconfig_test.go +++ b/plugins/inputs/logfile/fileconfig_test.go @@ -130,15 +130,34 @@ func TestTimestampParser(t *testing.T) { expectedTimestamp := time.Unix(1497882318, 0) timestampString := "19 Jun 2017 14:25:18" logEntry := fmt.Sprintf("%s [INFO] This is a test message.", timestampString) - timestamp := fileConfig.timestampFromLogLine(logEntry) + timestamp, modifiedLogEntry := fileConfig.timestampFromLogLine(logEntry) assert.Equal(t, expectedTimestamp.UnixNano(), timestamp.UnixNano(), fmt.Sprintf("The timestampFromLogLine value %v is not the same as expected %v.", timestamp, expectedTimestamp)) + assert.Equal(t, logEntry, modifiedLogEntry) // Test regex match for multiline, the first timestamp in multiline should be matched logEntry = fmt.Sprintf("%s [INFO] This is the first line.\n19 Jun 2017 14:25:19 [INFO] This is the second line.\n", timestampString) - timestamp = fileConfig.timestampFromLogLine(logEntry) + timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry) assert.Equal(t, expectedTimestamp.UnixNano(), timestamp.UnixNano(), fmt.Sprintf("The timestampFromLogLine value %v is not the same as expected %v.", timestamp, expectedTimestamp)) + assert.Equal(t, logEntry, modifiedLogEntry) + + // Test TrimTimeStamp for single line + fileConfig.TrimTimestamp = true + logEntry = fmt.Sprintf("%s [INFO] This is a test message.", timestampString) + trimmedTimestampString := " [INFO] This is a test message." + timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry) + assert.Equal(t, expectedTimestamp.UnixNano(), timestamp.UnixNano(), + fmt.Sprintf("The timestampFromLogLine value %v is not the same as expected %v.", timestamp, expectedTimestamp)) + assert.Equal(t, trimmedTimestampString, modifiedLogEntry) + + // Test TrimTimeStamp for multiline, the first timestamp in multiline should be matched + logEntry = fmt.Sprintf("%s [INFO] This is the first line.\n19 Jun 2017 14:25:19 [INFO] This is the second line.\n", timestampString) + trimmedTimestampString = " [INFO] This is the first line.\n19 Jun 2017 14:25:19 [INFO] This is the second line.\n" + timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry) + assert.Equal(t, expectedTimestamp.UnixNano(), timestamp.UnixNano(), + fmt.Sprintf("The timestampFromLogLine value %v is not the same as expected %v.", timestamp, expectedTimestamp)) + assert.Equal(t, trimmedTimestampString, modifiedLogEntry) } func TestTimestampParserWithPadding(t *testing.T) { @@ -155,15 +174,33 @@ func TestTimestampParserWithPadding(t *testing.T) { Timezone: timezone, TimezoneLoc: timezoneLoc} - logEntry := fmt.Sprintf(" 2 1 07:10:06 instance-id: i-02fce21a425a2efb3") - timestamp := fileConfig.timestampFromLogLine(logEntry) + logEntry := " 2 1 07:10:06 instance-id: i-02fce21a425a2efb3" + timestamp, modifiedLogEntry := fileConfig.timestampFromLogLine(logEntry) + assert.Equal(t, 7, timestamp.Hour(), fmt.Sprintf("Timestamp does not match: %v, act: %v", "7", timestamp.Hour())) + assert.Equal(t, 10, timestamp.Minute(), fmt.Sprintf("Timestamp does not match: %v, act: %v", "10", timestamp.Minute())) + assert.Equal(t, logEntry, modifiedLogEntry) + + logEntry = "2 1 07:10:06 instance-id: i-02fce21a425a2efb3" + timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry) + assert.Equal(t, 7, timestamp.Hour(), fmt.Sprintf("Timestamp does not match: %v, act: %v", "7", timestamp.Hour())) + assert.Equal(t, 10, timestamp.Minute(), fmt.Sprintf("Timestamp does not match: %v, act: %v", "10", timestamp.Minute())) + assert.Equal(t, logEntry, modifiedLogEntry) + + //Test when TrimTimeStamp is enabled + fileConfig.TrimTimestamp = true + logEntry = " 2 1 07:10:06 instance-id: i-02fce21a425a2efb3" + trimmedTimestampString := " instance-id: i-02fce21a425a2efb3" + timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry) assert.Equal(t, 7, timestamp.Hour(), fmt.Sprintf("Timestamp does not match: %v, act: %v", "7", timestamp.Hour())) assert.Equal(t, 10, timestamp.Minute(), fmt.Sprintf("Timestamp does not match: %v, act: %v", "10", timestamp.Minute())) + assert.Equal(t, trimmedTimestampString, modifiedLogEntry) - logEntry = fmt.Sprintf("2 1 07:10:06 instance-id: i-02fce21a425a2efb3") - timestamp = fileConfig.timestampFromLogLine(logEntry) + logEntry = "2 1 07:10:06 instance-id: i-02fce21a425a2efb3" + trimmedTimestampString = " instance-id: i-02fce21a425a2efb3" + timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry) assert.Equal(t, 7, timestamp.Hour(), fmt.Sprintf("Timestamp does not match: %v, act: %v", "7", timestamp.Hour())) assert.Equal(t, 10, timestamp.Minute(), fmt.Sprintf("Timestamp does not match: %v, act: %v", "10", timestamp.Minute())) + assert.Equal(t, trimmedTimestampString, modifiedLogEntry) } func TestTimestampParserDefault(t *testing.T) { @@ -183,26 +220,56 @@ func TestTimestampParserDefault(t *testing.T) { TimezoneLoc: timezoneLoc} // make sure layout is compatible for "Sep 9", "Sep 9" , "Sep 09", "Sep 09" options - logEntry := fmt.Sprintf("Sep 9 02:00:43 ip-10-4-213-132 \n") - timestamp := fileConfig.timestampFromLogLine(logEntry) + logEntry := "Sep 9 02:00:43 ip-10-4-213-132 \n" + timestamp, modifiedLogEntry := fileConfig.timestampFromLogLine(logEntry) + assert.Equal(t, 02, timestamp.Hour()) + assert.Equal(t, 00, timestamp.Minute()) + assert.Equal(t, logEntry, modifiedLogEntry) + + logEntry = "Sep 9 02:00:43 ip-10-4-213-132 \n" + timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry) assert.Equal(t, 02, timestamp.Hour()) assert.Equal(t, 00, timestamp.Minute()) + assert.Equal(t, logEntry, modifiedLogEntry) - logEntry = fmt.Sprintf("Sep 9 02:00:43 ip-10-4-213-132 \n") - timestamp = fileConfig.timestampFromLogLine(logEntry) + logEntry = "Sep 09 02:00:43 ip-10-4-213-132 \n" + timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry) assert.Equal(t, 02, timestamp.Hour()) assert.Equal(t, 00, timestamp.Minute()) + assert.Equal(t, logEntry, modifiedLogEntry) - logEntry = fmt.Sprintf("Sep 09 02:00:43 ip-10-4-213-132 \n") - timestamp = fileConfig.timestampFromLogLine(logEntry) + logEntry = "Sep 09 02:00:43 ip-10-4-213-132 \n" + timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry) assert.Equal(t, 02, timestamp.Hour()) assert.Equal(t, 00, timestamp.Minute()) + assert.Equal(t, logEntry, modifiedLogEntry) - logEntry = fmt.Sprintf("Sep 09 02:00:43 ip-10-4-213-132 \n") - timestamp = fileConfig.timestampFromLogLine(logEntry) + // When TrimTimestamp is enabled, make sure layout is compatible for "Sep 9", "Sep 9" , "Sep 09", "Sep 09" options and log value is trimmed correctly + fileConfig.TrimTimestamp = true + logEntry = "Sep 9 02:00:43 ip-10-4-213-132 \n" + trimmedTimestampString := " ip-10-4-213-132 \n" + timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry) assert.Equal(t, 02, timestamp.Hour()) assert.Equal(t, 00, timestamp.Minute()) + assert.Equal(t, trimmedTimestampString, modifiedLogEntry) + logEntry = "Sep 9 02:00:43 ip-10-4-213-132 \n" + timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry) + assert.Equal(t, 02, timestamp.Hour()) + assert.Equal(t, 00, timestamp.Minute()) + assert.Equal(t, trimmedTimestampString, modifiedLogEntry) + + logEntry = "Sep 09 02:00:43 ip-10-4-213-132 \n" + timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry) + assert.Equal(t, 02, timestamp.Hour()) + assert.Equal(t, 00, timestamp.Minute()) + assert.Equal(t, trimmedTimestampString, modifiedLogEntry) + + logEntry = "Sep 09 02:00:43 ip-10-4-213-132 \n" + timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry) + assert.Equal(t, 02, timestamp.Hour()) + assert.Equal(t, 00, timestamp.Minute()) + assert.Equal(t, trimmedTimestampString, modifiedLogEntry) } func TestTimestampParserWithFracSeconds(t *testing.T) { @@ -222,15 +289,34 @@ func TestTimestampParserWithFracSeconds(t *testing.T) { expectedTimestamp := time.Unix(1497882318, 234000000) timestampString := "19 Jun 2017 14:25:18,234088 UTC" logEntry := fmt.Sprintf("%s [INFO] This is a test message.", timestampString) - timestamp := fileConfig.timestampFromLogLine(logEntry) + timestamp, modifiedLogEntry := fileConfig.timestampFromLogLine(logEntry) assert.Equal(t, expectedTimestamp.UnixNano(), timestamp.UnixNano(), fmt.Sprintf("The timestampFromLogLine value %v is not the same as expected %v.", timestamp, expectedTimestamp)) + assert.Equal(t, logEntry, modifiedLogEntry) // Test regex match for multiline, the first timestamp in multiline should be matched logEntry = fmt.Sprintf("%s [INFO] This is the first line.\n19 Jun 2017 14:25:19,123456 UTC [INFO] This is the second line.\n", timestampString) - timestamp = fileConfig.timestampFromLogLine(logEntry) + timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry) + assert.Equal(t, expectedTimestamp.UnixNano(), timestamp.UnixNano(), + fmt.Sprintf("The timestampFromLogLine value %v is not the same as expected %v.", timestamp, expectedTimestamp)) + assert.Equal(t, logEntry, modifiedLogEntry) + + // Test TrimTimeStamp for single line + fileConfig.TrimTimestamp = true + logEntry = fmt.Sprintf("%s [INFO] This is a test message.", timestampString) + trimmedTimestampString := " [INFO] This is a test message." + timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry) + assert.Equal(t, expectedTimestamp.UnixNano(), timestamp.UnixNano(), + fmt.Sprintf("The timestampFromLogLine value %v is not the same as expected %v.", timestamp, expectedTimestamp)) + assert.Equal(t, trimmedTimestampString, modifiedLogEntry) + + // Test TrimTimeStamp for multiline, the first timestamp in multiline should be matched + logEntry = fmt.Sprintf("%s [INFO] This is the first line.\n19 Jun 2017 14:25:19,123456 UTC [INFO] This is the second line.\n", timestampString) + trimmedTimestampString = " [INFO] This is the first line.\n19 Jun 2017 14:25:19,123456 UTC [INFO] This is the second line.\n" + timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry) assert.Equal(t, expectedTimestamp.UnixNano(), timestamp.UnixNano(), fmt.Sprintf("The timestampFromLogLine value %v is not the same as expected %v.", timestamp, expectedTimestamp)) + assert.Equal(t, trimmedTimestampString, modifiedLogEntry) } func TestNonAllowlistedTimezone(t *testing.T) { diff --git a/plugins/inputs/logfile/logfile.go b/plugins/inputs/logfile/logfile.go index 1626957ebc..f73755f290 100644 --- a/plugins/inputs/logfile/logfile.go +++ b/plugins/inputs/logfile/logfile.go @@ -76,6 +76,7 @@ const sampleConfig = ` timestamp_regex = "^(\\d{2} \\w{3} \\d{4} \\d{2}:\\d{2}:\\d{2}).*$" timestamp_layout = ["_2 Jan 2006 15:04:05"] timezone = "UTC" + trim_timestamp = false multi_line_start_pattern = "{timestamp_regex}" ## Read file from beginning. from_beginning = false diff --git a/plugins/inputs/logfile/tailersrc.go b/plugins/inputs/logfile/tailersrc.go index df5b5967f9..466ae74562 100644 --- a/plugins/inputs/logfile/tailersrc.go +++ b/plugins/inputs/logfile/tailersrc.go @@ -67,7 +67,7 @@ type tailerSrc struct { stateFilePath string tailer *tail.Tail autoRemoval bool - timestampFn func(string) time.Time + timestampFn func(string) (time.Time, string) enc encoding.Encoding maxEventSize int truncateSuffix string @@ -91,7 +91,7 @@ func NewTailerSrc( autoRemoval bool, isMultilineStartFn func(string) bool, filters []*LogFilter, - timestampFn func(string) time.Time, + timestampFn func(string) (time.Time, string), enc encoding.Encoding, maxEventSize int, truncateSuffix string, @@ -195,9 +195,10 @@ func (ts *tailerSrc) runTail() { if !ok { if msgBuf.Len() > 0 { msg := msgBuf.String() + timestamp, modifiedMsg := ts.timestampFn(msg) e := &LogEvent{ - msg: msg, - t: ts.timestampFn(msg), + msg: modifiedMsg, + t: timestamp, offset: *fo, src: ts, } @@ -249,9 +250,10 @@ func (ts *tailerSrc) runTail() { if msgBuf.Len() > 0 { msg := msgBuf.String() + timestamp, modifiedMsg := ts.timestampFn(msg) e := &LogEvent{ - msg: msg, - t: ts.timestampFn(msg), + msg: modifiedMsg, + t: timestamp, offset: *fo, src: ts, } @@ -276,9 +278,10 @@ func (ts *tailerSrc) runTail() { } msg := msgBuf.String() + timestamp, modifiedMsg := ts.timestampFn(msg) e := &LogEvent{ - msg: msg, - t: ts.timestampFn(msg), + msg: modifiedMsg, + t: timestamp, offset: *fo, src: ts, } diff --git a/plugins/inputs/logfile/tailersrc_test.go b/plugins/inputs/logfile/tailersrc_test.go index 23a8ae8ba4..c29ea9aab5 100644 --- a/plugins/inputs/logfile/tailersrc_test.go +++ b/plugins/inputs/logfile/tailersrc_test.go @@ -324,7 +324,7 @@ func TestTailerSrcFiltersMultiLineLogs(t *testing.T) { assertExpectedLogsPublished(t, n, int(*resources.consumed)) } -func parseRFC3339Timestamp(line string) time.Time { +func parseRFC3339Timestamp(line string) (time.Time, string) { // Use RFC3339 for testing `2006-01-02T15:04:05Z07:00` re := regexp.MustCompile(`\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[Z+\-]\d{2}:\d{2}`) tstr := re.FindString(line) @@ -332,7 +332,7 @@ func parseRFC3339Timestamp(line string) time.Time { if tstr != "" { t, _ = time.Parse(time.RFC3339, tstr) } - return t + return t, line } func logLine(s string, l int, t time.Time) string { diff --git a/translator/config/schema.json b/translator/config/schema.json index 4938baf571..83f51b8f3b 100644 --- a/translator/config/schema.json +++ b/translator/config/schema.json @@ -959,6 +959,10 @@ "UTC" ] }, + "trim_timestamp" : { + "type": "boolean", + "description": "Whether to trim the timestamp in the log message" + }, "encoding": { "type": "string", "minLength": 1, diff --git a/translator/tocwconfig/sampleConfig/complete_linux_config.conf b/translator/tocwconfig/sampleConfig/complete_linux_config.conf index aef5f442db..12730d434c 100644 --- a/translator/tocwconfig/sampleConfig/complete_linux_config.conf +++ b/translator/tocwconfig/sampleConfig/complete_linux_config.conf @@ -54,6 +54,7 @@ pipe = false retention_in_days = 5 timezone = "UTC" + trim_timestamp = true [[inputs.logfile.file_config]] auto_removal = true diff --git a/translator/tocwconfig/sampleConfig/complete_linux_config.json b/translator/tocwconfig/sampleConfig/complete_linux_config.json index c2fb70e151..48062c0e20 100755 --- a/translator/tocwconfig/sampleConfig/complete_linux_config.json +++ b/translator/tocwconfig/sampleConfig/complete_linux_config.json @@ -258,6 +258,7 @@ "log_group_name": "amazon-cloudwatch-agent.log", "log_stream_name": "amazon-cloudwatch-agent.log", "timezone": "UTC", + "trim_timestamp": true, "retention_in_days": 5 }, { diff --git a/translator/tocwconfig/totomlconfig/testdata/agentToml.conf b/translator/tocwconfig/totomlconfig/testdata/agentToml.conf index 4b9a68b829..4a75a90a5b 100644 --- a/translator/tocwconfig/totomlconfig/testdata/agentToml.conf +++ b/translator/tocwconfig/totomlconfig/testdata/agentToml.conf @@ -54,6 +54,7 @@ pipe = false retention_in_days = 5 timezone = "UTC" + trim_timestamp = true [[inputs.logfile.file_config]] auto_removal = true diff --git a/translator/tocwconfig/totomlconfig/testdata/agentToml.json b/translator/tocwconfig/totomlconfig/testdata/agentToml.json index bba1c17499..206d3e8fb0 100644 --- a/translator/tocwconfig/totomlconfig/testdata/agentToml.json +++ b/translator/tocwconfig/totomlconfig/testdata/agentToml.json @@ -182,7 +182,8 @@ "log_group_name": "amazon-cloudwatch-agent.log", "log_stream_name": "amazon-cloudwatch-agent.log", "timezone": "UTC", - "retention_in_days": 5 + "retention_in_days": 5, + "trim_timestamp": true }, { "file_path": "/opt/aws/amazon-cloudwatch-agent/logs/test.log", diff --git a/translator/tocwconfig/totomlconfig/tomlConfigTemplate/tomlConfig.go b/translator/tocwconfig/totomlconfig/tomlConfigTemplate/tomlConfig.go index 2a26ad9bb2..b93f3832c8 100644 --- a/translator/tocwconfig/totomlconfig/tomlConfigTemplate/tomlConfig.go +++ b/translator/tocwconfig/totomlconfig/tomlConfigTemplate/tomlConfig.go @@ -126,6 +126,8 @@ type ( Pipe bool RetentionInDays int `toml:"retention_in_days"` Timezone string + //Customer specifies if the timestamp from the log message should be trimmed + TrimTimestamp bool `toml:"trim_timestamp"` //Customer specified service.name ServiceName string `toml:"service_name"` //Customer specified deployment.environment diff --git a/translator/translate/logs/logs_collected/files/collect_list/ruleTrimTimestamp.go b/translator/translate/logs/logs_collected/files/collect_list/ruleTrimTimestamp.go new file mode 100644 index 0000000000..6cd4ec8a4b --- /dev/null +++ b/translator/translate/logs/logs_collected/files/collect_list/ruleTrimTimestamp.go @@ -0,0 +1,33 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package collect_list //nolint:revive + +import ( + "github.com/aws/amazon-cloudwatch-agent/translator" +) + +const TrimTimestampSectionKey = "trim_timestamp" + +type TrimTimestamp struct { +} + +func (r *TrimTimestamp) ApplyRule(input interface{}) (string, interface{}) { + _, val := translator.DefaultCase(TrimTimestampSectionKey, "", input) + if val == "" { + return "", "" + } + + boolVal, ok := val.(bool) + if !ok { + return TrimTimestampSectionKey, false + } + + return TrimTimestampSectionKey, boolVal +} + +func init() { + l := new(TrimTimestamp) + r := []Rule{l} + RegisterRule(TrimTimestampSectionKey, r) +} From 95b6d2761fcabe114e650a9e94e44eb2e58e6033 Mon Sep 17 00:00:00 2001 From: POOJA REDDY NATHALA Date: Fri, 28 Feb 2025 12:06:22 -0500 Subject: [PATCH 2/2] Trim leading whitespaces after trimming timestamp from Logs (#1574) --- plugins/inputs/logfile/fileconfig.go | 5 +++-- plugins/inputs/logfile/fileconfig_test.go | 20 +++++++++++++------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/plugins/inputs/logfile/fileconfig.go b/plugins/inputs/logfile/fileconfig.go index 23a17cc96c..fe0b8d8d17 100644 --- a/plugins/inputs/logfile/fileconfig.go +++ b/plugins/inputs/logfile/fileconfig.go @@ -211,8 +211,9 @@ func (config *FileConfig) timestampFromLogLine(logValue string) (time.Time, stri } } if config.TrimTimestamp { - // Trim the entire timestamp portion (from start to end of the match) - return timestamp, logValue[:index[0]] + logValue[index[1]:] + // Trim the entire timestamp portion and leading whitespaces + // The whitespace characters being removed are: space, tab, newline, and carriage return ( " \t\n\r") + return timestamp, strings.TrimLeft(logValue[:index[0]]+logValue[index[1]:], " \t\n\r") } return timestamp, logValue } diff --git a/plugins/inputs/logfile/fileconfig_test.go b/plugins/inputs/logfile/fileconfig_test.go index f0c974e266..5315fb182a 100644 --- a/plugins/inputs/logfile/fileconfig_test.go +++ b/plugins/inputs/logfile/fileconfig_test.go @@ -145,7 +145,7 @@ func TestTimestampParser(t *testing.T) { // Test TrimTimeStamp for single line fileConfig.TrimTimestamp = true logEntry = fmt.Sprintf("%s [INFO] This is a test message.", timestampString) - trimmedTimestampString := " [INFO] This is a test message." + trimmedTimestampString := "[INFO] This is a test message." timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry) assert.Equal(t, expectedTimestamp.UnixNano(), timestamp.UnixNano(), fmt.Sprintf("The timestampFromLogLine value %v is not the same as expected %v.", timestamp, expectedTimestamp)) @@ -153,7 +153,7 @@ func TestTimestampParser(t *testing.T) { // Test TrimTimeStamp for multiline, the first timestamp in multiline should be matched logEntry = fmt.Sprintf("%s [INFO] This is the first line.\n19 Jun 2017 14:25:19 [INFO] This is the second line.\n", timestampString) - trimmedTimestampString = " [INFO] This is the first line.\n19 Jun 2017 14:25:19 [INFO] This is the second line.\n" + trimmedTimestampString = "[INFO] This is the first line.\n19 Jun 2017 14:25:19 [INFO] This is the second line.\n" timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry) assert.Equal(t, expectedTimestamp.UnixNano(), timestamp.UnixNano(), fmt.Sprintf("The timestampFromLogLine value %v is not the same as expected %v.", timestamp, expectedTimestamp)) @@ -189,14 +189,20 @@ func TestTimestampParserWithPadding(t *testing.T) { //Test when TrimTimeStamp is enabled fileConfig.TrimTimestamp = true logEntry = " 2 1 07:10:06 instance-id: i-02fce21a425a2efb3" - trimmedTimestampString := " instance-id: i-02fce21a425a2efb3" + trimmedTimestampString := "instance-id: i-02fce21a425a2efb3" timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry) assert.Equal(t, 7, timestamp.Hour(), fmt.Sprintf("Timestamp does not match: %v, act: %v", "7", timestamp.Hour())) assert.Equal(t, 10, timestamp.Minute(), fmt.Sprintf("Timestamp does not match: %v, act: %v", "10", timestamp.Minute())) assert.Equal(t, trimmedTimestampString, modifiedLogEntry) logEntry = "2 1 07:10:06 instance-id: i-02fce21a425a2efb3" - trimmedTimestampString = " instance-id: i-02fce21a425a2efb3" + timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry) + assert.Equal(t, 7, timestamp.Hour(), fmt.Sprintf("Timestamp does not match: %v, act: %v", "7", timestamp.Hour())) + assert.Equal(t, 10, timestamp.Minute(), fmt.Sprintf("Timestamp does not match: %v, act: %v", "10", timestamp.Minute())) + assert.Equal(t, trimmedTimestampString, modifiedLogEntry) + + logEntry = " instance-id: i-02fce21a425a2efb3 2 1 07:10:06" + trimmedTimestampString = "instance-id: i-02fce21a425a2efb3 " timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry) assert.Equal(t, 7, timestamp.Hour(), fmt.Sprintf("Timestamp does not match: %v, act: %v", "7", timestamp.Hour())) assert.Equal(t, 10, timestamp.Minute(), fmt.Sprintf("Timestamp does not match: %v, act: %v", "10", timestamp.Minute())) @@ -247,7 +253,7 @@ func TestTimestampParserDefault(t *testing.T) { // When TrimTimestamp is enabled, make sure layout is compatible for "Sep 9", "Sep 9" , "Sep 09", "Sep 09" options and log value is trimmed correctly fileConfig.TrimTimestamp = true logEntry = "Sep 9 02:00:43 ip-10-4-213-132 \n" - trimmedTimestampString := " ip-10-4-213-132 \n" + trimmedTimestampString := "ip-10-4-213-132 \n" timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry) assert.Equal(t, 02, timestamp.Hour()) assert.Equal(t, 00, timestamp.Minute()) @@ -304,7 +310,7 @@ func TestTimestampParserWithFracSeconds(t *testing.T) { // Test TrimTimeStamp for single line fileConfig.TrimTimestamp = true logEntry = fmt.Sprintf("%s [INFO] This is a test message.", timestampString) - trimmedTimestampString := " [INFO] This is a test message." + trimmedTimestampString := "[INFO] This is a test message." timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry) assert.Equal(t, expectedTimestamp.UnixNano(), timestamp.UnixNano(), fmt.Sprintf("The timestampFromLogLine value %v is not the same as expected %v.", timestamp, expectedTimestamp)) @@ -312,7 +318,7 @@ func TestTimestampParserWithFracSeconds(t *testing.T) { // Test TrimTimeStamp for multiline, the first timestamp in multiline should be matched logEntry = fmt.Sprintf("%s [INFO] This is the first line.\n19 Jun 2017 14:25:19,123456 UTC [INFO] This is the second line.\n", timestampString) - trimmedTimestampString = " [INFO] This is the first line.\n19 Jun 2017 14:25:19,123456 UTC [INFO] This is the second line.\n" + trimmedTimestampString = "[INFO] This is the first line.\n19 Jun 2017 14:25:19,123456 UTC [INFO] This is the second line.\n" timestamp, modifiedLogEntry = fileConfig.timestampFromLogLine(logEntry) assert.Equal(t, expectedTimestamp.UnixNano(), timestamp.UnixNano(), fmt.Sprintf("The timestampFromLogLine value %v is not the same as expected %v.", timestamp, expectedTimestamp))