Replace generator plugin with file input in read mode

internal/app/daemon/run/run.go

@@ -12,8 +12,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/breml/logstash-config/ast"
-	"github.com/breml/logstash-config/ast/astutil"
 	"github.com/imkira/go-observer"
 	"github.com/pkg/errors"
 	"github.com/tidwall/gjson"
@@ -165,7 +163,6 @@ func (s Test) Run() (err error) {
 		if err != nil {
 			return err
 		}
-		s.validateInputLines(t.InputLines)
 
 		result, err := c.ExecuteTest(context.Background(), &pb.ExecuteTestRequest{
 			SessionID:      sessionID,
@@ -254,17 +251,6 @@ func (s Test) createImplicitPipeline() (string, error) {
 	return pipelineFile, nil
 }
 
-func (s Test) validateInputLines(lines []string) {
-	for _, line := range lines {
-		_, doubleQuoteErr := astutil.Quote(line, ast.DoubleQuoted)
-		_, singleQuoteErr := astutil.Quote(line, ast.SingleQuoted)
-
-		if doubleQuoteErr != nil && singleQuoteErr != nil {
-			s.log.Warningf("Test input line %q contains unescaped double and single quotes, single quotes will be escaped automatically", line)
-		}
-	}
-}
-
 func (s Test) postProcessResults(results []string, t testcase.TestCaseSet) ([]string, error) {
 	var err error
 

internal/daemon/session/files.go

@@ -21,13 +21,17 @@ output {
 
 const inputGenerator = `
 input {
-  generator {
-    lines => [
-      {{ .InputLines }}
-    ]
+  file {
+    id => '__lfv_file_input'
+    path => "{{ .InputFilename }}"
     {{ .InputCodec }}
-    count => 1
-    threads => 1
+    mode => "read"
+    file_completed_action => "log"
+    file_completed_log_path => "{{ .InputFilename }}.log"
+    exit_after_read => true
+    {{ if .RandomDelimiter }}
+    delimiter => "xyTY1zS2mwJ9xuFCIkrPucLtiSuYIkXAmgCXB142"
+    {{ end }}
   }
 }
 
@@ -43,7 +47,7 @@ filter {
   mutate {
     # Remove fields "host", "sequence" and optionally "message", which are
     # automatically created by the generator input.
-    remove_field => [ "host", "sequence" ]
+    remove_field => [ "host", "path", "[@metadata][host]", "[@metadata][path]" ]
   }
 
   translate {

internal/daemon/session/session.go

@@ -6,11 +6,10 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strconv"
 	"strings"
 
-	"github.com/breml/logstash-config/ast"
-	"github.com/breml/logstash-config/ast/astutil"
 	"github.com/pkg/errors"
 
 	"github.com/magnusbaeck/logstash-filter-verifier/v2/internal/daemon/idgen"
@@ -182,8 +181,18 @@ func (s *Session) ExecuteTest(inputPlugin string, inputLines []string, inEvents
 		return err
 	}
 
+	input := []byte(strings.Join(inputLines, "\n"))
+	if len(input) == 0 || input[len(input)-1] != '\n' {
+		input = append(input, byte('\n'))
+	}
+	inputFilename := filepath.Join(inputDir, "input.lines")
+	err = os.WriteFile(inputFilename, input, 0600)
+	if err != nil {
+		return err
+	}
+
 	pipelineFilename := filepath.Join(inputDir, "input.conf")
-	err = createInput(pipelineFilename, fieldsFilename, inputPluginName, inputLines, inputCodec)
+	err = createInput(pipelineFilename, fieldsFilename, inputPluginName, inputFilename, inputCodec)
 	if err != nil {
 		return err
 	}
@@ -225,26 +234,25 @@ func prepareFields(fieldsFilename string, inEvents []map[string]interface{}) err
 	return nil
 }
 
-func createInput(pipelineFilename string, fieldsFilename string, inputPluginName string, inputLines []string, inputCodec string) error {
-	for i := range inputLines {
-		var err error
-		inputLine, err := astutil.Quote(inputLines[i], ast.DoubleQuoted)
-		if err != nil {
-			inputLine = astutil.QuoteWithEscape(inputLines[i], ast.SingleQuoted)
-		}
-		inputLines[i] = inputLine
-	}
+var delimitedCodecs = regexp.MustCompile("codec => ((edn|json)_lines|graphite|(multi)?line)")
 
+func createInput(pipelineFilename string, fieldsFilename string, inputPluginName string, inputFilename string, inputCodec string) error {
+	randomDelimiter := false
+	if delimitedCodecs.MatchString(inputCodec) {
+		randomDelimiter = true
+	}
 	templateData := struct {
 		InputPluginName          string
-		InputLines               string
+		InputFilename            string
 		InputCodec               string
+		RandomDelimiter          bool
 		FieldsFilename           string
 		DummyEventInputIndicator string
 	}{
 		InputPluginName:          inputPluginName,
-		InputLines:               strings.Join(inputLines, ", "),
+		InputFilename:            inputFilename,
 		InputCodec:               inputCodec,
+		RandomDelimiter:          randomDelimiter,
 		FieldsFilename:           fieldsFilename,
 		DummyEventInputIndicator: testcase.DummyEventInputIndicator,
 	}

testdata/drop_and_split/drop_and_split.conf

@@ -14,6 +14,7 @@ filter {
   if [message] =~ /split/ {
     split {
       id => "split"
+      terminator => "|"
       # field => "results"
     }
   }

testdata/testcases/codec_test/edn_lines.json

@@ -6,10 +6,10 @@
   "testcases": [
     {
       "input": [
-        "{:test \"foo\", :value 0}\n",
-        "{:test \"bar\", :value 1}\n",
-        "{:test \"baz\", :value 2}\n",
-        "{:test \"foo2\", :value 3}\n"
+        "{:test \"foo\", :value 0}",
+        "{:test \"bar\", :value 1}",
+        "{:test \"baz\", :value 2}",
+        "{:test \"foo2\", :value 3}"
       ],
       "expected": [
         {

testdata/testcases/codec_test/graphite.json

@@ -3,8 +3,8 @@
   "testcases": [
     {
       "input": [
-        "foo.bar.metric1 100 1617385070\n",
-        "foo.bar.metric2 200 1617385070\n"
+        "foo.bar.metric1 100 1617385070",
+        "foo.bar.metric2 200 1617385070"
       ],
       "expected": [
         {

testdata/testcases/codec_test/json_lines.json

@@ -6,9 +6,9 @@
   "testcases": [
     {
       "input": [
-        "{\"test\": \"foo\", \"value\": 0}\n",
-        "{\"test\": \"bar\", \"value\": 1}\n",
-        "{\"test\": \"baz\", \"value\": 2}\n",
+        "{\"test\": \"foo\", \"value\": 0}",
+        "{\"test\": \"bar\", \"value\": 1}",
+        "{\"test\": \"baz\", \"value\": 2}",
         "{\"test\": \"foo2\", \"value\": 3}"
       ],
       "expected": [

testdata/testcases/codec_test/line.json

@@ -6,8 +6,8 @@
   "testcases": [
     {
       "input": [
-        "test case message\n",
-        "test case message 2\n"
+        "test case message",
+        "test case message 2"
       ],
       "expected": [
         {

testdata/testcases/codec_test/multiline.json

@@ -6,8 +6,10 @@
   "testcases": [
     {
       "input": [
-        "foo\n  bar\n  baz\n",
-        "other foo\n  bar\n  baz\n"
+        "foo",
+        "  bar",
+        "  baz",
+        "other foo\n  bar\n  baz"
       ],
       "expected": [
         {

testdata/testcases/drop_and_split/split.json

@@ -9,7 +9,7 @@
     {
       "input": [
         "normal input",
-        "split this input\ninto multiple events",
+        "split this input|into multiple events",
         "normal input again"
       ],
       "expected": [

testdata/testcases/special_chars/testcases.json

@@ -6,7 +6,7 @@
   "testcases": [
     {
       "input": [
-        "message with \r \n \t \\ (special characters)",
+        "message with \t \\ (special characters)",
         "message with only \" (double quote)",
         "message with only ' (single quote)",
         "message with \" and ' (double and single quote)",
@@ -16,7 +16,7 @@
       ],
       "expected": [
         {
-          "message": "message with \r \n \t \\ (special characters)"
+          "message": "message with \t \\ (special characters)"
         },
         {
           "message": "message with only \" (double quote)"
@@ -25,7 +25,7 @@
           "message": "message with only ' (single quote)"
         },
         {
-          "message": "message with \" and \\' (double and single quote)"
+          "message": "message with \" and ' (double and single quote)"
         },
         {
           "message": "message with \\\" and ' (escaped double and single quote)"