diff --git a/CHANGELOG-6.md b/CHANGELOG-6.md index 5643e29bf..bd275f9c2 100644 --- a/CHANGELOG-6.md +++ b/CHANGELOG-6.md @@ -16,6 +16,17 @@ Versioning](http://semver.org/spec/v2.0.0.html). ### Added - Adding a flag at agent level to avoid collecting system.networks property in the agent entity state - Added silences sorting by expiration to GraphQL service +- Added log-millisecond-timestamps backend configuration flag +- Added a session store, used to detect and prevent refresh token reuse +- Added support escape special characters in InfluxDB Line + +### Changed +- Log handler error at error level instead of info level +- Users are now automatically logged out after a period of inactivity (12h) + +## [6.9.2] - 2023-03-08 + +### Added - Added GraphQL validator for query node depth ## [6.9.1] - 2022-12-01 diff --git a/agent/transformers/influx_db.go b/agent/transformers/influx_db.go index 65254ef27..d65b49480 100644 --- a/agent/transformers/influx_db.go +++ b/agent/transformers/influx_db.go @@ -15,10 +15,10 @@ type InfluxList []Influx // Influx contains values of influx db line output metric format type Influx struct { - Measurement string - TagSet []*v2.MetricTag - FieldSet []*Field - Timestamp int64 + Measurement string + TagSet []*v2.MetricTag + FieldSet []*Field + Timestamp int64 } // Transform transforms a metric in influx db line protocol to Sensu Metric @@ -28,10 +28,10 @@ func (i InfluxList) Transform() []*v2.MetricPoint { for _, influx := range i { for _, fieldSet := range influx.FieldSet { mp := &v2.MetricPoint{ - Name: influx.Measurement + "." + fieldSet.Key, - Value: fieldSet.Value, - Timestamp: influx.Timestamp, - Tags: influx.TagSet, + Name: influx.Measurement + "." + fieldSet.Key, + Value: fieldSet.Value, + Timestamp: influx.Timestamp, + Tags: influx.TagSet, } points = append(points, mp) } @@ -43,8 +43,8 @@ func (i InfluxList) Transform() []*v2.MetricPoint { func ParseInflux(event *v2.Event) InfluxList { var influxList InfluxList fields := logrus.Fields{ - "namespace": event.Check.Namespace, - "check": event.Check.Name, + "namespace": event.Check.Namespace, + "check": event.Check.Name, } metric := strings.TrimSpace(event.Check.Output) @@ -57,28 +57,28 @@ OUTER: fields["line"] = l l++ i := Influx{} - args := strings.Split(line, " ") + args := splitWithoutEscaped(line, " ") if len(args) != 3 && len(args) != 2 { logger.WithFields(fields).WithError(ErrMetricExtraction).Error("influxdb line format requires 2 arguments with a 3rd (optional) timestamp") continue } - measurementTag := strings.Split(args[0], ",") - i.Measurement = measurementTag[0] + measurementTag := splitWithoutEscaped(args[0], ",") + i.Measurement = unescape(measurementTag[0]) tagList := []*v2.MetricTag{} if len(measurementTag) == 1 { i.TagSet = tagList } else { for i, tagSet := range measurementTag { if i != 0 { - ts := strings.Split(tagSet, "=") + ts := splitWithoutEscaped(tagSet, "=") if len(ts) != 2 { logger.WithFields(fields).WithError(ErrMetricExtraction).Error("metric tag set is invalid, must contain a key=value pair") continue OUTER } tag := &v2.MetricTag{ - Name: ts[0], - Value: ts[1], + Name: unescape(ts[0]), + Value: unescape(ts[1]), } tagList = append(tagList, tag) } @@ -90,10 +90,10 @@ OUTER: i.TagSet = append(i.TagSet, event.Check.OutputMetricTags...) } - fieldSets := strings.Split(args[1], ",") + fieldSets := splitWithoutEscaped(args[1], ",") fieldList := []*Field{} for _, fieldSet := range fieldSets { - fs := strings.Split(fieldSet, "=") + fs := splitWithoutEscaped(fieldSet, "=") if len(fs) != 2 { logger.WithFields(fields).WithError(ErrMetricExtraction).Error("metric field set is invalid, must contain a key=value pair") continue OUTER @@ -104,8 +104,8 @@ OUTER: continue OUTER } field := &Field{ - Key: fs[0], - Value: f, + Key: unescape(fs[0]), + Value: f, } fieldList = append(fieldList, field) } @@ -133,3 +133,19 @@ OUTER: return influxList } + +func splitWithoutEscaped(s, sep string) []string { + s = strings.ReplaceAll(s, `\`+sep, "\x00") + segments := strings.Split(s, sep) + for i := range segments { + segments[i] = strings.ReplaceAll(segments[i], "\x00", sep) + } + return segments +} + +func unescape(s string) string { + s = strings.ReplaceAll(s, `\\`, "\x00") + s = strings.ReplaceAll(s, `\`, "") + s = strings.ReplaceAll(s, "\x00", `\`) + return s +} diff --git a/agent/transformers/influx_db_test.go b/agent/transformers/influx_db_test.go index 51ea6c7d3..f9793b714 100644 --- a/agent/transformers/influx_db_test.go +++ b/agent/transformers/influx_db_test.go @@ -11,143 +11,183 @@ func TestParseInflux(t *testing.T) { assert := assert.New(t) testCases := []struct { - metric string - expectedFormat InfluxList - timeInconclusive bool + metric string + expectedFormat InfluxList + timeInconclusive bool }{ { - metric: "weather,location=us-midwest,season=summer temperature=82,humidity=30 1465839830100400200", + metric: "weather,location=us-midwest,season=summer temperature=82,humidity=30 1465839830100400200", expectedFormat: InfluxList{ { - Measurement: "weather", + Measurement: "weather", TagSet: []*v2.MetricTag{ { - Name: "location", - Value: "us-midwest", + Name: "location", + Value: "us-midwest", }, { - Name: "season", - Value: "summer", + Name: "season", + Value: "summer", }, }, FieldSet: []*Field{ { - Key: "temperature", - Value: 82, + Key: "temperature", + Value: 82, }, { - Key: "humidity", - Value: 30, + Key: "humidity", + Value: 30, }, }, - Timestamp: 1465839830, + Timestamp: 1465839830, }, }, }, { - metric: "weather,location=us-midwest,season=summer temperature=82,humidity=30 1465839830100400200\nweather temperature=82 1465839830100400200\n", + metric: "weather,location=us-midwest,season=summer temperature=82,humidity=30 1465839830100400200\nweather temperature=82 1465839830100400200\n", expectedFormat: InfluxList{ { - Measurement: "weather", + Measurement: "weather", TagSet: []*v2.MetricTag{ { - Name: "location", - Value: "us-midwest", + Name: "location", + Value: "us-midwest", }, { - Name: "season", - Value: "summer", + Name: "season", + Value: "summer", }, }, FieldSet: []*Field{ { - Key: "temperature", - Value: 82, + Key: "temperature", + Value: 82, }, { - Key: "humidity", - Value: 30, + Key: "humidity", + Value: 30, }, }, - Timestamp: 1465839830, + Timestamp: 1465839830, }, { - Measurement: "weather", - TagSet: []*v2.MetricTag{}, + Measurement: "weather", + TagSet: []*v2.MetricTag{}, FieldSet: []*Field{ { - Key: "temperature", - Value: 82, + Key: "temperature", + Value: 82, }, }, - Timestamp: 1465839830, + Timestamp: 1465839830, }, }, }, { - metric: "weather,location=us-midwest,season=summer temperature=82,humidity=30 1465839830100400200\nfoo\n", + metric: "weather,location=us-midwest,season=summer temperature=82,humidity=30 1465839830100400200\nfoo\n", expectedFormat: InfluxList{ { - Measurement: "weather", + Measurement: "weather", TagSet: []*v2.MetricTag{ { - Name: "location", - Value: "us-midwest", + Name: "location", + Value: "us-midwest", }, { - Name: "season", - Value: "summer", + Name: "season", + Value: "summer", }, }, FieldSet: []*Field{ { - Key: "temperature", - Value: 82, + Key: "temperature", + Value: 82, }, { - Key: "humidity", - Value: 30, + Key: "humidity", + Value: 30, }, }, - Timestamp: 1465839830, + Timestamp: 1465839830, }, }, }, { - metric: "weather temperature=82 1465839830100400200", + metric: "weather temperature=82 1465839830100400200", expectedFormat: InfluxList{ { - Measurement: "weather", - TagSet: []*v2.MetricTag{}, + Measurement: "weather", + TagSet: []*v2.MetricTag{}, FieldSet: []*Field{ { - Key: "temperature", - Value: 82, + Key: "temperature", + Value: 82, }, }, - Timestamp: 1465839830, + Timestamp: 1465839830, }, }, }, { - metric: "weather temperature=82", - timeInconclusive: true, + metric: "wea\\ ther,locat\\,ion=us-mid\\=west,sea\"son=sum\\\\mer te\\ mp\\,er\\=at\"ure=82,h\\ um\\,id\\=it\"y=30 1465839830100400200\nw\\ e\\,a\\=t\"her te\\ mp\\,er\\=at\"ure=82 1465839830100400200\n", + expectedFormat: InfluxList{ + { + Measurement: "wea ther", + TagSet: []*v2.MetricTag{ + { + Name: "locat,ion", + Value: "us-mid=west", + }, + { + Name: `sea"son`, + Value: `sum\mer`, + }, + }, + FieldSet: []*Field{ + { + Key: `te mp,er=at"ure`, + Value: 82, + }, + { + Key: `h um,id=it"y`, + Value: 30, + }, + }, + Timestamp: 1465839830, + }, + { + Measurement: `w e,a=t"her`, + TagSet: []*v2.MetricTag{}, + FieldSet: []*Field{ + { + Key: `te mp,er=at"ure`, + Value: 82, + }, + }, + Timestamp: 1465839830, + }, + }, + }, + { + metric: "weather temperature=82", + timeInconclusive: true, }, { - metric: "weather temperature=82 12345 blah", - expectedFormat: InfluxList(nil), + metric: "weather temperature=82 12345 blah", + expectedFormat: InfluxList(nil), }, { - metric: "weather,location temperature= 1465839830100400200", - expectedFormat: InfluxList(nil), + metric: "weather,location temperature= 1465839830100400200", + expectedFormat: InfluxList(nil), }, { - metric: "", - expectedFormat: InfluxList(nil), + metric: "", + expectedFormat: InfluxList(nil), }, { - metric: "foo bar baz", - expectedFormat: InfluxList(nil), + metric: "foo bar baz", + expectedFormat: InfluxList(nil), }, } @@ -167,192 +207,192 @@ func TestParseInfluxTags(t *testing.T) { assert := assert.New(t) testCases := []struct { - metric string - expectedFormat InfluxList - timeInconclusive bool - outputMetricTags []*v2.MetricTag + metric string + expectedFormat InfluxList + timeInconclusive bool + outputMetricTags []*v2.MetricTag }{ { - metric: "weather,location=us-midwest,season=summer temperature=82,humidity=30 1465839830100400200", + metric: "weather,location=us-midwest,season=summer temperature=82,humidity=30 1465839830100400200", expectedFormat: InfluxList{ { - Measurement: "weather", + Measurement: "weather", TagSet: []*v2.MetricTag{ { - Name: "location", - Value: "us-midwest", + Name: "location", + Value: "us-midwest", }, { - Name: "season", - Value: "summer", + Name: "season", + Value: "summer", }, { - Name: "instance", - Value: "hostname", + Name: "instance", + Value: "hostname", }, }, FieldSet: []*Field{ { - Key: "temperature", - Value: 82, + Key: "temperature", + Value: 82, }, { - Key: "humidity", - Value: 30, + Key: "humidity", + Value: 30, }, }, - Timestamp: 1465839830, + Timestamp: 1465839830, }, }, outputMetricTags: []*v2.MetricTag{ { - Name: "instance", - Value: "hostname", + Name: "instance", + Value: "hostname", }, }, }, { - metric: "weather,location=us-midwest,season=summer temperature=82,humidity=30 1465839830100400200", + metric: "weather,location=us-midwest,season=summer temperature=82,humidity=30 1465839830100400200", expectedFormat: InfluxList{ { - Measurement: "weather", + Measurement: "weather", TagSet: []*v2.MetricTag{ { - Name: "location", - Value: "us-midwest", + Name: "location", + Value: "us-midwest", }, { - Name: "season", - Value: "summer", + Name: "season", + Value: "summer", }, }, FieldSet: []*Field{ { - Key: "temperature", - Value: 82, + Key: "temperature", + Value: 82, }, { - Key: "humidity", - Value: 30, + Key: "humidity", + Value: 30, }, }, - Timestamp: 1465839830, + Timestamp: 1465839830, }, }, }, { - metric: "weather,location=us-midwest,season=summer temperature=82,humidity=30 1465839830100400200", + metric: "weather,location=us-midwest,season=summer temperature=82,humidity=30 1465839830100400200", expectedFormat: InfluxList{ { - Measurement: "weather", + Measurement: "weather", TagSet: []*v2.MetricTag{ { - Name: "location", - Value: "us-midwest", + Name: "location", + Value: "us-midwest", }, { - Name: "season", - Value: "summer", + Name: "season", + Value: "summer", }, }, FieldSet: []*Field{ { - Key: "temperature", - Value: 82, + Key: "temperature", + Value: 82, }, { - Key: "humidity", - Value: 30, + Key: "humidity", + Value: 30, }, }, - Timestamp: 1465839830, + Timestamp: 1465839830, }, }, - outputMetricTags: []*v2.MetricTag{}, + outputMetricTags: []*v2.MetricTag{}, }, { - metric: "weather,location=us-midwest,season=summer temperature=82,humidity=30 1465839830100400200", + metric: "weather,location=us-midwest,season=summer temperature=82,humidity=30 1465839830100400200", expectedFormat: InfluxList{ { - Measurement: "weather", + Measurement: "weather", TagSet: []*v2.MetricTag{ { - Name: "location", - Value: "us-midwest", + Name: "location", + Value: "us-midwest", }, { - Name: "season", - Value: "summer", + Name: "season", + Value: "summer", }, { - Name: "foo", - Value: "bar", + Name: "foo", + Value: "bar", }, { - Name: "boo", - Value: "baz", + Name: "boo", + Value: "baz", }, }, FieldSet: []*Field{ { - Key: "temperature", - Value: 82, + Key: "temperature", + Value: 82, }, { - Key: "humidity", - Value: 30, + Key: "humidity", + Value: 30, }, }, - Timestamp: 1465839830, + Timestamp: 1465839830, }, }, outputMetricTags: []*v2.MetricTag{ { - Name: "foo", - Value: "bar", + Name: "foo", + Value: "bar", }, { - Name: "boo", - Value: "baz", + Name: "boo", + Value: "baz", }, }, }, { - metric: "weather,location=us-midwest,season=summer temperature=82,humidity=30 1465839830100400200", + metric: "weather,location=us-midwest,season=summer temperature=82,humidity=30 1465839830100400200", expectedFormat: InfluxList{ { - Measurement: "weather", + Measurement: "weather", TagSet: []*v2.MetricTag{ { - Name: "location", - Value: "us-midwest", + Name: "location", + Value: "us-midwest", }, { - Name: "season", - Value: "summer", + Name: "season", + Value: "summer", }, { - Name: "", - Value: "", + Name: "", + Value: "", }, }, FieldSet: []*Field{ { - Key: "temperature", - Value: 82, + Key: "temperature", + Value: 82, }, { - Key: "humidity", - Value: 30, + Key: "humidity", + Value: 30, }, }, - Timestamp: 1465839830, + Timestamp: 1465839830, }, }, outputMetricTags: []*v2.MetricTag{ { - Name: "", - Value: "", + Name: "", + Value: "", }, }, }, @@ -375,64 +415,64 @@ func TestTransformInflux(t *testing.T) { assert := assert.New(t) testCases := []struct { - metric InfluxList - expectedFormat []*v2.MetricPoint + metric InfluxList + expectedFormat []*v2.MetricPoint }{ { metric: InfluxList{ { - Measurement: "weather", + Measurement: "weather", TagSet: []*v2.MetricTag{ { - Name: "location", - Value: "us-midwest", + Name: "location", + Value: "us-midwest", }, { - Name: "season", - Value: "summer", + Name: "season", + Value: "summer", }, }, FieldSet: []*Field{ { - Key: "temperature", - Value: 82, + Key: "temperature", + Value: 82, }, { - Key: "humidity", - Value: 30, + Key: "humidity", + Value: 30, }, }, - Timestamp: 1465839830, + Timestamp: 1465839830, }, }, expectedFormat: []*v2.MetricPoint{ { - Name: "weather.temperature", - Value: 82, - Timestamp: 1465839830, + Name: "weather.temperature", + Value: 82, + Timestamp: 1465839830, Tags: []*v2.MetricTag{ { - Name: "location", - Value: "us-midwest", + Name: "location", + Value: "us-midwest", }, { - Name: "season", - Value: "summer", + Name: "season", + Value: "summer", }, }, }, { - Name: "weather.humidity", - Value: 30, - Timestamp: 1465839830, + Name: "weather.humidity", + Value: 30, + Timestamp: 1465839830, Tags: []*v2.MetricTag{ { - Name: "location", - Value: "us-midwest", + Name: "location", + Value: "us-midwest", }, { - Name: "season", - Value: "summer", + Name: "season", + Value: "summer", }, }, }, @@ -441,13 +481,13 @@ func TestTransformInflux(t *testing.T) { { metric: InfluxList{ { - Measurement: "", - TagSet: []*v2.MetricTag{}, - FieldSet: []*Field{}, - Timestamp: 0, + Measurement: "", + TagSet: []*v2.MetricTag{}, + FieldSet: []*Field{}, + Timestamp: 0, }, }, - expectedFormat: []*v2.MetricPoint(nil), + expectedFormat: []*v2.MetricPoint(nil), }, } @@ -463,127 +503,127 @@ func TestParseAndTransformInflux(t *testing.T) { assert := assert.New(t) testCases := []struct { - metric string - expectedFormat []*v2.MetricPoint - timeInconclusive bool + metric string + expectedFormat []*v2.MetricPoint + timeInconclusive bool }{ { - metric: "weather,location=us-midwest,season=summer temperature=82,humidity=30 1465839830100400200", + metric: "weather,location=us-midwest,season=summer temperature=82,humidity=30 1465839830100400200", expectedFormat: []*v2.MetricPoint{ { - Name: "weather.temperature", - Value: 82, - Timestamp: 1465839830, + Name: "weather.temperature", + Value: 82, + Timestamp: 1465839830, Tags: []*v2.MetricTag{ { - Name: "location", - Value: "us-midwest", + Name: "location", + Value: "us-midwest", }, { - Name: "season", - Value: "summer", + Name: "season", + Value: "summer", }, }, }, { - Name: "weather.humidity", - Value: 30, - Timestamp: 1465839830, + Name: "weather.humidity", + Value: 30, + Timestamp: 1465839830, Tags: []*v2.MetricTag{ { - Name: "location", - Value: "us-midwest", + Name: "location", + Value: "us-midwest", }, { - Name: "season", - Value: "summer", + Name: "season", + Value: "summer", }, }, }, }, }, { - metric: "weather temperature=82,humidity=30 1465839830100400200\nfoo", + metric: "weather temperature=82,humidity=30 1465839830100400200\nfoo", expectedFormat: []*v2.MetricPoint{ { - Name: "weather.temperature", - Value: 82, - Timestamp: 1465839830, - Tags: []*v2.MetricTag{}, + Name: "weather.temperature", + Value: 82, + Timestamp: 1465839830, + Tags: []*v2.MetricTag{}, }, { - Name: "weather.humidity", - Value: 30, - Timestamp: 1465839830, - Tags: []*v2.MetricTag{}, + Name: "weather.humidity", + Value: 30, + Timestamp: 1465839830, + Tags: []*v2.MetricTag{}, }, }, }, { - metric: "weather,location=us-midwest,season=summer temperature=82 1465839830100400200\nweather,location=us-midwest,season=summer humidity=30 1465839830100400200", + metric: "weather,location=us-midwest,season=summer temperature=82 1465839830100400200\nweather,location=us-midwest,season=summer humidity=30 1465839830100400200", expectedFormat: []*v2.MetricPoint{ { - Name: "weather.temperature", - Value: 82, - Timestamp: 1465839830, + Name: "weather.temperature", + Value: 82, + Timestamp: 1465839830, Tags: []*v2.MetricTag{ { - Name: "location", - Value: "us-midwest", + Name: "location", + Value: "us-midwest", }, { - Name: "season", - Value: "summer", + Name: "season", + Value: "summer", }, }, }, { - Name: "weather.humidity", - Value: 30, - Timestamp: 1465839830, + Name: "weather.humidity", + Value: 30, + Timestamp: 1465839830, Tags: []*v2.MetricTag{ { - Name: "location", - Value: "us-midwest", + Name: "location", + Value: "us-midwest", }, { - Name: "season", - Value: "summer", + Name: "season", + Value: "summer", }, }, }, }, }, { - metric: "metric value=0 0\n", + metric: "metric value=0 0\n", expectedFormat: []*v2.MetricPoint{ { - Name: "metric.value", - Value: 0, - Timestamp: 0, - Tags: []*v2.MetricTag{}, + Name: "metric.value", + Value: 0, + Timestamp: 0, + Tags: []*v2.MetricTag{}, }, }, }, { - metric: "weather temperature=82", - timeInconclusive: true, + metric: "weather temperature=82", + timeInconclusive: true, }, { - metric: "weather temperature=82 12345 blah", - expectedFormat: []*v2.MetricPoint(nil), + metric: "weather temperature=82 12345 blah", + expectedFormat: []*v2.MetricPoint(nil), }, { - metric: "weather,location temperature= 1465839830100400200", - expectedFormat: []*v2.MetricPoint(nil), + metric: "weather,location temperature= 1465839830100400200", + expectedFormat: []*v2.MetricPoint(nil), }, { - metric: "", - expectedFormat: []*v2.MetricPoint(nil), + metric: "", + expectedFormat: []*v2.MetricPoint(nil), }, { - metric: "foo bar baz", - expectedFormat: []*v2.MetricPoint(nil), + metric: "foo bar baz", + expectedFormat: []*v2.MetricPoint(nil), }, }