Only add fields if present

Fixes: magnusbaeck#147
breml · Nov 1, 2021 · bd14ec5 · bd14ec5
1 parent 7ab46ae
commit bd14ec5
Show file tree

Hide file tree

Showing 5 changed files with 47 additions and 11 deletions.
diff --git a/internal/daemon/session/files.go b/internal/daemon/session/files.go
@@ -50,6 +50,7 @@ filter {
     remove_field => [ "host", "path", "[@metadata][host]", "[@metadata][path]" ]
   }
 
+{{ if .HasFields }}
   translate {
     dictionary_path => "{{ .FieldsFilename }}"
     field => "[@metadata][__lfv_id]"
@@ -59,6 +60,7 @@ filter {
     fallback => "__lfv_fields_not_found"
     refresh_interval => 0
   }
+{{ end }}
 
   ruby {
     id => '__lfv_ruby_fields'

diff --git a/internal/daemon/session/session.go b/internal/daemon/session/session.go
@@ -176,7 +176,7 @@ func (s *Session) ExecuteTest(inputPlugin string, inputLines []string, inEvents
 	}
 
 	fieldsFilename := filepath.Join(inputDir, "fields.json")
-	err = prepareFields(fieldsFilename, inEvents)
+	hasFields, err := prepareFields(fieldsFilename, inEvents)
 	if err != nil {
 		return err
 	}
@@ -192,7 +192,7 @@ func (s *Session) ExecuteTest(inputPlugin string, inputLines []string, inEvents
 	}
 
 	pipelineFilename := filepath.Join(inputDir, "input.conf")
-	err = createInput(pipelineFilename, fieldsFilename, inputPluginName, inputFilename, inputCodec)
+	err = createInput(pipelineFilename, hasFields, fieldsFilename, inputPluginName, inputFilename, inputCodec)
 	if err != nil {
 		return err
 	}
@@ -214,29 +214,34 @@ func (s *Session) ExecuteTest(inputPlugin string, inputLines []string, inEvents
 	return nil
 }
 
-func prepareFields(fieldsFilename string, inEvents []map[string]interface{}) error {
+func prepareFields(fieldsFilename string, inEvents []map[string]interface{}) (bool, error) {
+	hasFields := false
+
 	fields := make(map[string]map[string]interface{})
 
 	for i, event := range inEvents {
 		id := fmt.Sprintf("%d", i)
 		fields[id] = event
+		if len(event) > 0 {
+			hasFields = true
+		}
 	}
 
 	bfields, err := json.Marshal(fields)
 	if err != nil {
-		return err
+		return false, err
 	}
 	err = ioutil.WriteFile(fieldsFilename, bfields, 0600)
 	if err != nil {
-		return err
+		return false, err
 	}
 
-	return nil
+	return hasFields, nil
 }
 
 var delimitedCodecs = regexp.MustCompile("codec => ((edn|json)_lines|graphite|(multi)?line)")
 
-func createInput(pipelineFilename string, fieldsFilename string, inputPluginName string, inputFilename string, inputCodec string) error {
+func createInput(pipelineFilename string, hasFields bool, fieldsFilename string, inputPluginName string, inputFilename string, inputCodec string) error {
 	randomDelimiter := false
 	if delimitedCodecs.MatchString(inputCodec) {
 		randomDelimiter = true
@@ -246,13 +251,15 @@ func createInput(pipelineFilename string, fieldsFilename string, inputPluginName
 		InputFilename            string
 		InputCodec               string
 		RandomDelimiter          bool
+		HasFields                bool
 		FieldsFilename           string
 		DummyEventInputIndicator string
 	}{
 		InputPluginName:          inputPluginName,
 		InputFilename:            inputFilename,
 		InputCodec:               inputCodec,
 		RandomDelimiter:          randomDelimiter,
+		HasFields:                hasFields,
 		FieldsFilename:           fieldsFilename,
 		DummyEventInputIndicator: testcase.DummyEventInputIndicator,
 	}

diff --git a/internal/testcase/testcase.go b/internal/testcase/testcase.go
@@ -207,7 +207,14 @@ func New(reader io.Reader, configType string) (*TestCaseSet, error) {
 
 	tcs.descriptions = make([]string, len(tcs.ExpectedEvents))
 
-	for range tcs.InputLines {
+	maxEvents := len(tcs.ExpectedEvents)
+	for _, tc := range tcs.TestCases {
+		maxEvents += len(tc.ExpectedEvents)
+	}
+	if len(tcs.InputLines) > maxEvents {
+		maxEvents = len(tcs.InputLines)
+	}
+	for i := 0; i < maxEvents; i++ {
 		tcs.Events = append(tcs.Events, tcs.InputFields.Clone())
 	}
 

diff --git a/testdata/testcases/codec_test/json.json b/testdata/testcases/codec_test/json.json
@@ -6,9 +6,7 @@
   "testcases": [
     {
       "input": [
-        "[ {\"test\": \"foo\", \"value\": 0},{\"test\": \"bar\", \"value\": 1},{\"test\": \"baz\", \"value\": 2} ]",
-        "",
-        ""
+        "[ {\"test\": \"foo\", \"value\": 0},{\"test\": \"bar\", \"value\": 1},{\"test\": \"baz\", \"value\": 2} ]"
       ],
       "expected": [
         {

diff --git a/testdata/testcases/codec_test/multiline.yaml b/testdata/testcases/codec_test/multiline.yaml
@@ -0,0 +1,22 @@
+# issue #147
+input_plugin: "multiline"
+fields:
+  test: true
+ignore:
+  - "@timestamp"
+testcases:
+  - input:
+      - |
+        This is message 1 and
+          this is message 1 also
+        This is message 2 and
+          this is message 2 also
+    expected:
+      - input_codec: "multiline"
+        message: "This is message 1 and\n  this is message 1 also"
+        tags: ["multiline", "input_codec_multiline"]
+        test: true
+      - input_codec: "multiline"
+        message: "This is message 2 and\n  this is message 2 also"
+        tags: ["multiline", "input_codec_multiline"]
+        test: true