From bd14ec5307e5e71b45cf8a715d734a4aa3039869 Mon Sep 17 00:00:00 2001 From: Lucas Bremgartner Date: Fri, 22 Oct 2021 19:45:51 +0200 Subject: [PATCH] Only add fields if present Fixes: #147 --- internal/daemon/session/files.go | 2 ++ internal/daemon/session/session.go | 21 ++++++++++++------- internal/testcase/testcase.go | 9 +++++++- testdata/testcases/codec_test/json.json | 4 +--- testdata/testcases/codec_test/multiline.yaml | 22 ++++++++++++++++++++ 5 files changed, 47 insertions(+), 11 deletions(-) create mode 100644 testdata/testcases/codec_test/multiline.yaml diff --git a/internal/daemon/session/files.go b/internal/daemon/session/files.go index d8fcf39..57a2dcd 100644 --- a/internal/daemon/session/files.go +++ b/internal/daemon/session/files.go @@ -50,6 +50,7 @@ filter { remove_field => [ "host", "path", "[@metadata][host]", "[@metadata][path]" ] } +{{ if .HasFields }} translate { dictionary_path => "{{ .FieldsFilename }}" field => "[@metadata][__lfv_id]" @@ -59,6 +60,7 @@ filter { fallback => "__lfv_fields_not_found" refresh_interval => 0 } +{{ end }} ruby { id => '__lfv_ruby_fields' diff --git a/internal/daemon/session/session.go b/internal/daemon/session/session.go index 234dfe3..26aba72 100644 --- a/internal/daemon/session/session.go +++ b/internal/daemon/session/session.go @@ -176,7 +176,7 @@ func (s *Session) ExecuteTest(inputPlugin string, inputLines []string, inEvents } fieldsFilename := filepath.Join(inputDir, "fields.json") - err = prepareFields(fieldsFilename, inEvents) + hasFields, err := prepareFields(fieldsFilename, inEvents) if err != nil { return err } @@ -192,7 +192,7 @@ func (s *Session) ExecuteTest(inputPlugin string, inputLines []string, inEvents } pipelineFilename := filepath.Join(inputDir, "input.conf") - err = createInput(pipelineFilename, fieldsFilename, inputPluginName, inputFilename, inputCodec) + err = createInput(pipelineFilename, hasFields, fieldsFilename, inputPluginName, inputFilename, inputCodec) if err != nil { return err } @@ -214,29 +214,34 @@ func (s *Session) ExecuteTest(inputPlugin string, inputLines []string, inEvents return nil } -func prepareFields(fieldsFilename string, inEvents []map[string]interface{}) error { +func prepareFields(fieldsFilename string, inEvents []map[string]interface{}) (bool, error) { + hasFields := false + fields := make(map[string]map[string]interface{}) for i, event := range inEvents { id := fmt.Sprintf("%d", i) fields[id] = event + if len(event) > 0 { + hasFields = true + } } bfields, err := json.Marshal(fields) if err != nil { - return err + return false, err } err = ioutil.WriteFile(fieldsFilename, bfields, 0600) if err != nil { - return err + return false, err } - return nil + return hasFields, nil } var delimitedCodecs = regexp.MustCompile("codec => ((edn|json)_lines|graphite|(multi)?line)") -func createInput(pipelineFilename string, fieldsFilename string, inputPluginName string, inputFilename string, inputCodec string) error { +func createInput(pipelineFilename string, hasFields bool, fieldsFilename string, inputPluginName string, inputFilename string, inputCodec string) error { randomDelimiter := false if delimitedCodecs.MatchString(inputCodec) { randomDelimiter = true @@ -246,6 +251,7 @@ func createInput(pipelineFilename string, fieldsFilename string, inputPluginName InputFilename string InputCodec string RandomDelimiter bool + HasFields bool FieldsFilename string DummyEventInputIndicator string }{ @@ -253,6 +259,7 @@ func createInput(pipelineFilename string, fieldsFilename string, inputPluginName InputFilename: inputFilename, InputCodec: inputCodec, RandomDelimiter: randomDelimiter, + HasFields: hasFields, FieldsFilename: fieldsFilename, DummyEventInputIndicator: testcase.DummyEventInputIndicator, } diff --git a/internal/testcase/testcase.go b/internal/testcase/testcase.go index 6e0153e..f345c98 100644 --- a/internal/testcase/testcase.go +++ b/internal/testcase/testcase.go @@ -207,7 +207,14 @@ func New(reader io.Reader, configType string) (*TestCaseSet, error) { tcs.descriptions = make([]string, len(tcs.ExpectedEvents)) - for range tcs.InputLines { + maxEvents := len(tcs.ExpectedEvents) + for _, tc := range tcs.TestCases { + maxEvents += len(tc.ExpectedEvents) + } + if len(tcs.InputLines) > maxEvents { + maxEvents = len(tcs.InputLines) + } + for i := 0; i < maxEvents; i++ { tcs.Events = append(tcs.Events, tcs.InputFields.Clone()) } diff --git a/testdata/testcases/codec_test/json.json b/testdata/testcases/codec_test/json.json index ea87f15..838aff9 100644 --- a/testdata/testcases/codec_test/json.json +++ b/testdata/testcases/codec_test/json.json @@ -6,9 +6,7 @@ "testcases": [ { "input": [ - "[ {\"test\": \"foo\", \"value\": 0},{\"test\": \"bar\", \"value\": 1},{\"test\": \"baz\", \"value\": 2} ]", - "", - "" + "[ {\"test\": \"foo\", \"value\": 0},{\"test\": \"bar\", \"value\": 1},{\"test\": \"baz\", \"value\": 2} ]" ], "expected": [ { diff --git a/testdata/testcases/codec_test/multiline.yaml b/testdata/testcases/codec_test/multiline.yaml new file mode 100644 index 0000000..d3e0f49 --- /dev/null +++ b/testdata/testcases/codec_test/multiline.yaml @@ -0,0 +1,22 @@ +# issue #147 +input_plugin: "multiline" +fields: + test: true +ignore: + - "@timestamp" +testcases: + - input: + - | + This is message 1 and + this is message 1 also + This is message 2 and + this is message 2 also + expected: + - input_codec: "multiline" + message: "This is message 1 and\n this is message 1 also" + tags: ["multiline", "input_codec_multiline"] + test: true + - input_codec: "multiline" + message: "This is message 2 and\n this is message 2 also" + tags: ["multiline", "input_codec_multiline"] + test: true