Fix pipelines.yml issues (#125)

* Fix pipeline.id with dash
* Fix path.config pointing to dir in pipelines.yml

Solves issue mentioned in: #93 (comment)

internal/daemon/instance/logstash/processors.go

@@ -116,7 +116,8 @@ func extractPipelines(in string) []string {
 	}
 	pipelines := strings.Split(in[1:len(in)-1], ",")
 	for i := range pipelines {
-		pipelines[i] = strings.Trim(pipelines[i], " :")
+		pipelines[i] = strings.TrimLeft(pipelines[i], " :")
+		pipelines[i] = strings.Trim(pipelines[i], `"`)
 	}
 	return pipelines
 }

internal/daemon/instance/logstash/processors_test.go

@@ -0,0 +1,38 @@
+package logstash
+
+import (
+	"testing"
+
+	"github.com/matryer/is"
+)
+
+func TestExtractPipelines(t *testing.T) {
+	cases := []struct {
+		name  string
+		input string
+
+		want []string
+	}{
+		{
+			name:  "single word pipeline id",
+			input: `[:output, :stdin, :lfv_output_stdout, :lfv_ukPSsPZk_main, :lfv_input_1]`,
+
+			want: []string{"output", "stdin", "lfv_output_stdout", "lfv_ukPSsPZk_main", "lfv_input_1"},
+		},
+		{
+			name:  "pipeline id with dash",
+			input: `[:output, :stdin, :lfv_output_stdout, :"lfv_ukPSsPZk_main-test", :lfv_input_1]`,
+
+			want: []string{"output", "stdin", "lfv_output_stdout", "lfv_ukPSsPZk_main-test", "lfv_input_1"},
+		},
+	}
+
+	for _, test := range cases {
+		t.Run(test.name, func(t *testing.T) {
+			is := is.New(t)
+
+			got := extractPipelines(test.input)
+			is.Equal(test.want, got)
+		})
+	}
+}

internal/daemon/pipeline/pipeline.go

@@ -77,6 +77,9 @@ func processNestedKeys(pipelines Pipelines) {
 func (a Archive) Validate(addMissingID bool) error {
 	var inputs, outputs int
 	for _, pipeline := range a.Pipelines {
+		if strings.HasSuffix(pipeline.Config, "/") {
+			pipeline.Config += "*"
+		}
 		configFilepath := filepath.Join(a.BasePath, pipeline.Config)
 		if filepath.IsAbs(pipeline.Config) {
 			configFilepath = pipeline.Config
@@ -86,6 +89,13 @@ func (a Archive) Validate(addMissingID bool) error {
 			return err
 		}
 		for _, file := range files {
+			fi, err := os.Stat(file)
+			if err != nil {
+				return err
+			}
+			if fi.IsDir() {
+				continue
+			}
 			var relFile string
 			if path.IsAbs(a.BasePath) {
 				relFile = strings.TrimPrefix(file, a.BasePath)
@@ -141,6 +151,9 @@ func (a Archive) ZipWithPreprocessor(preprocess func([]byte) ([]byte, error)) ([
 	}
 
 	for _, pipeline := range a.Pipelines {
+		if strings.HasSuffix(pipeline.Config, "/") {
+			pipeline.Config += "*"
+		}
 		configFilepath := filepath.Join(a.BasePath, pipeline.Config)
 		if filepath.IsAbs(pipeline.Config) {
 			configFilepath = pipeline.Config
@@ -150,6 +163,13 @@ func (a Archive) ZipWithPreprocessor(preprocess func([]byte) ([]byte, error)) ([
 			return nil, err
 		}
 		for _, file := range files {
+			fi, err := os.Stat(file)
+			if err != nil {
+				return nil, err
+			}
+			if fi.IsDir() {
+				continue
+			}
 			var relFile string
 			if path.IsAbs(a.BasePath) {
 				relFile = strings.TrimPrefix(file, a.BasePath)

internal/daemon/pipeline/pipeline_test.go

@@ -130,6 +130,13 @@ func TestZip(t *testing.T) {
 
 			wantFiles: 2,
 		},
+		{
+			name:     "success basic pipeline dir name",
+			pipeline: "testdata/pipelines_basic_dir_name.yml",
+			basePath: "testdata/",
+
+			wantFiles: 2,
+		},
 		{
 			name:     "success advanced pipeline",
 			pipeline: "testdata/pipelines_advanced.yml",

internal/daemon/pipeline/testdata/pipelines_basic_dir_name.yml

@@ -0,0 +1,4 @@
+---
+
+- pipeline.id: main
+  path.config: "folder/"

testdata/basic_pipeline_debug.yml

@@ -1,2 +1,2 @@
-- pipeline.id: main
-  path.config: "main/**/*.conf"
+- pipeline.id: main-debug
+  path.config: "main/**/*"

testdata/conditional_output.yml

@@ -1,2 +1,2 @@
 - pipeline.id: main
-  path.config: "main/*.conf"
+  path.config: "main/"