From f22e89f907f26c8b943177c79a73732eb38a0007 Mon Sep 17 00:00:00 2001
From: Lucas Bremgartner <breml@users.noreply.github.com>
Date: Wed, 8 Sep 2021 18:13:10 +0200
Subject: [PATCH] Fix pipelines.yml issues (#125)

* Fix pipeline.id with dash
* Fix path.config pointing to dir in pipelines.yml

Solves issue mentioned in: https://github.com/magnusbaeck/logstash-filter-verifier/issues/93#issuecomment-913725182
---
 .../daemon/instance/logstash/processors.go    |  3 +-
 .../instance/logstash/processors_test.go      | 38 +++++++++++++++++++
 internal/daemon/pipeline/pipeline.go          | 20 ++++++++++
 internal/daemon/pipeline/pipeline_test.go     |  7 ++++
 .../testdata/pipelines_basic_dir_name.yml     |  4 ++
 testdata/basic_pipeline_debug.yml             |  4 +-
 testdata/conditional_output.yml               |  2 +-
 7 files changed, 74 insertions(+), 4 deletions(-)
 create mode 100644 internal/daemon/instance/logstash/processors_test.go
 create mode 100644 internal/daemon/pipeline/testdata/pipelines_basic_dir_name.yml

diff --git a/internal/daemon/instance/logstash/processors.go b/internal/daemon/instance/logstash/processors.go
index 6bef4e2..bed8d40 100644
--- a/internal/daemon/instance/logstash/processors.go
+++ b/internal/daemon/instance/logstash/processors.go
@@ -116,7 +116,8 @@ func extractPipelines(in string) []string {
 	}
 	pipelines := strings.Split(in[1:len(in)-1], ",")
 	for i := range pipelines {
-		pipelines[i] = strings.Trim(pipelines[i], " :")
+		pipelines[i] = strings.TrimLeft(pipelines[i], " :")
+		pipelines[i] = strings.Trim(pipelines[i], `"`)
 	}
 	return pipelines
 }
diff --git a/internal/daemon/instance/logstash/processors_test.go b/internal/daemon/instance/logstash/processors_test.go
new file mode 100644
index 0000000..1bd2f7b
--- /dev/null
+++ b/internal/daemon/instance/logstash/processors_test.go
@@ -0,0 +1,38 @@
+package logstash
+
+import (
+	"testing"
+
+	"github.com/matryer/is"
+)
+
+func TestExtractPipelines(t *testing.T) {
+	cases := []struct {
+		name  string
+		input string
+
+		want []string
+	}{
+		{
+			name:  "single word pipeline id",
+			input: `[:output, :stdin, :lfv_output_stdout, :lfv_ukPSsPZk_main, :lfv_input_1]`,
+
+			want: []string{"output", "stdin", "lfv_output_stdout", "lfv_ukPSsPZk_main", "lfv_input_1"},
+		},
+		{
+			name:  "pipeline id with dash",
+			input: `[:output, :stdin, :lfv_output_stdout, :"lfv_ukPSsPZk_main-test", :lfv_input_1]`,
+
+			want: []string{"output", "stdin", "lfv_output_stdout", "lfv_ukPSsPZk_main-test", "lfv_input_1"},
+		},
+	}
+
+	for _, test := range cases {
+		t.Run(test.name, func(t *testing.T) {
+			is := is.New(t)
+
+			got := extractPipelines(test.input)
+			is.Equal(test.want, got)
+		})
+	}
+}
diff --git a/internal/daemon/pipeline/pipeline.go b/internal/daemon/pipeline/pipeline.go
index 169cee3..444e0d2 100644
--- a/internal/daemon/pipeline/pipeline.go
+++ b/internal/daemon/pipeline/pipeline.go
@@ -77,6 +77,9 @@ func processNestedKeys(pipelines Pipelines) {
 func (a Archive) Validate(addMissingID bool) error {
 	var inputs, outputs int
 	for _, pipeline := range a.Pipelines {
+		if strings.HasSuffix(pipeline.Config, "/") {
+			pipeline.Config += "*"
+		}
 		configFilepath := filepath.Join(a.BasePath, pipeline.Config)
 		if filepath.IsAbs(pipeline.Config) {
 			configFilepath = pipeline.Config
@@ -86,6 +89,13 @@ func (a Archive) Validate(addMissingID bool) error {
 			return err
 		}
 		for _, file := range files {
+			fi, err := os.Stat(file)
+			if err != nil {
+				return err
+			}
+			if fi.IsDir() {
+				continue
+			}
 			var relFile string
 			if path.IsAbs(a.BasePath) {
 				relFile = strings.TrimPrefix(file, a.BasePath)
@@ -141,6 +151,9 @@ func (a Archive) ZipWithPreprocessor(preprocess func([]byte) ([]byte, error)) ([
 	}
 
 	for _, pipeline := range a.Pipelines {
+		if strings.HasSuffix(pipeline.Config, "/") {
+			pipeline.Config += "*"
+		}
 		configFilepath := filepath.Join(a.BasePath, pipeline.Config)
 		if filepath.IsAbs(pipeline.Config) {
 			configFilepath = pipeline.Config
@@ -150,6 +163,13 @@ func (a Archive) ZipWithPreprocessor(preprocess func([]byte) ([]byte, error)) ([
 			return nil, err
 		}
 		for _, file := range files {
+			fi, err := os.Stat(file)
+			if err != nil {
+				return nil, err
+			}
+			if fi.IsDir() {
+				continue
+			}
 			var relFile string
 			if path.IsAbs(a.BasePath) {
 				relFile = strings.TrimPrefix(file, a.BasePath)
diff --git a/internal/daemon/pipeline/pipeline_test.go b/internal/daemon/pipeline/pipeline_test.go
index 58672ba..d7a8600 100644
--- a/internal/daemon/pipeline/pipeline_test.go
+++ b/internal/daemon/pipeline/pipeline_test.go
@@ -130,6 +130,13 @@ func TestZip(t *testing.T) {
 
 			wantFiles: 2,
 		},
+		{
+			name:     "success basic pipeline dir name",
+			pipeline: "testdata/pipelines_basic_dir_name.yml",
+			basePath: "testdata/",
+
+			wantFiles: 2,
+		},
 		{
 			name:     "success advanced pipeline",
 			pipeline: "testdata/pipelines_advanced.yml",
diff --git a/internal/daemon/pipeline/testdata/pipelines_basic_dir_name.yml b/internal/daemon/pipeline/testdata/pipelines_basic_dir_name.yml
new file mode 100644
index 0000000..e4378d2
--- /dev/null
+++ b/internal/daemon/pipeline/testdata/pipelines_basic_dir_name.yml
@@ -0,0 +1,4 @@
+---
+
+- pipeline.id: main
+  path.config: "folder/"
diff --git a/testdata/basic_pipeline_debug.yml b/testdata/basic_pipeline_debug.yml
index a9f230e..f8a90c7 100644
--- a/testdata/basic_pipeline_debug.yml
+++ b/testdata/basic_pipeline_debug.yml
@@ -1,2 +1,2 @@
-- pipeline.id: main
-  path.config: "main/**/*.conf"
+- pipeline.id: main-debug
+  path.config: "main/**/*"
diff --git a/testdata/conditional_output.yml b/testdata/conditional_output.yml
index 3b10926..932d836 100644
--- a/testdata/conditional_output.yml
+++ b/testdata/conditional_output.yml
@@ -1,2 +1,2 @@
 - pipeline.id: main
-  path.config: "main/*.conf"
+  path.config: "main/"