From 6e5e608023d1c2aab7f07d9df11beb08b7ecc3c9 Mon Sep 17 00:00:00 2001 From: Fabian Holler Date: Mon, 25 Mar 2024 17:02:26 +0100 Subject: [PATCH] support defining environment variables for executed command Support defining environment variables that are set when the command of a task executed. These can be defined in the new Task.Environment.variables array in the .app.toml or in task include configuration files. Defined environment variables are automatically tracked as task inputs. --- internal/command/show.go | 2 ++ pkg/baur/inputresolver.go | 40 +++++++++++++++++++++++----------- pkg/baur/inputresolver_test.go | 29 ++++++++++++++++++++++++ pkg/baur/task.go | 28 +++++++++++++----------- pkg/baur/taskrunner.go | 2 ++ pkg/baur/taskrunner_test.go | 11 ++++++++++ pkg/cfg/app.go | 3 +++ pkg/cfg/app_test.go | 4 +++- pkg/cfg/environment.go | 20 +++++++++++++++++ pkg/cfg/includedb_test.go | 4 ++++ pkg/cfg/task.go | 16 +++++++++----- pkg/cfg/taskdef.go | 5 +++++ pkg/cfg/taskinclude.go | 16 +++++++++----- 13 files changed, 143 insertions(+), 37 deletions(-) create mode 100644 pkg/cfg/environment.go diff --git a/internal/command/show.go b/internal/command/show.go index 97741b8e5..6a082d437 100644 --- a/internal/command/show.go +++ b/internal/command/show.go @@ -186,6 +186,8 @@ func (c *showCmd) printTask(formatter Formatter, task *baur.Task) { c.strCmd(task.Command), ), "", "") + mustWriteStringSliceRows(formatter, "Environment Variables:", 1, task.EnvironmentVariables) + if task.HasInputs() { mustWriteRow(formatter, "", "", "", "") mustWriteRow(formatter, "", term.Underline("Inputs:"), "", "") diff --git a/pkg/baur/inputresolver.go b/pkg/baur/inputresolver.go index f8c5f855c..1ebacefe1 100644 --- a/pkg/baur/inputresolver.go +++ b/pkg/baur/inputresolver.go @@ -97,16 +97,20 @@ func (i *InputResolver) Resolve(ctx context.Context, task *Task) ([]Input, error return nil, err } - envVars, err := i.resolveEnvVarInputs(task.UnresolvedInputs.EnvironmentVariables) + envVarInputs, err := i.resolveEnvVarInputs(task.UnresolvedInputs.EnvironmentVariables) if err != nil { return nil, fmt.Errorf("resolving environment variable inputs failed: %w", err) } + if err := envVarSliceMapSet(envVarInputs, task.EnvironmentVariables); err != nil { + return nil, fmt.Errorf("converting Environment.variables to map failed: %w", err) + } + stats := i.cache.Statistics() log.Debugf("inputresolver: cache statistic: %d entries, %d hits, %d miss, ratio %.2f%%\n", stats.Entries, stats.Hits, stats.Miss, stats.HitRatio()) - return append(uniqInputs, envVarMapToInputslice(envVars)...), nil + return append(uniqInputs, envVarMapToInputslice(envVarInputs)...), nil } func (i *InputResolver) resolveCacheFileGlob(path string, optional bool) ([]string, error) { @@ -397,6 +401,19 @@ func (i *InputResolver) queryGitTrackedDb(ctx context.Context, absPath string) ( return nil, "", fmt.Errorf("got unsupport git.TrackedObject mode: %o", obj.Mode) } +func envVarSliceMapSet(m map[string]string, envVars []string) error { + for _, kv := range envVars { + k, v, found := strings.Cut(kv, "=") + if !found { + return fmt.Errorf("%q does not contain a '=' character", kv) + } + + m[k] = v + } + + return nil +} + func (i *InputResolver) setEnvVars() { if i.environmentVariables != nil { return @@ -405,16 +422,12 @@ func (i *InputResolver) setEnvVars() { // os.Environ() does not return env variables that are declared but undefined. // environment variables that have an empty string assigned are returned. environ := os.Environ() - i.environmentVariables = make(map[string]string, len(environ)) - for _, env := range environ { - k, v, found := strings.Cut(env, "=") - if !found { - // impossible scenario - panic(fmt.Sprintf("element %q returned by os.Environ() does not contain a '=' character", env)) - } - - i.environmentVariables[k] = v + i.environmentVariables = make(map[string]string, len(environ)) + err := envVarSliceMapSet(i.environmentVariables, environ) + if err != nil { + // impossible scenario + panic("BUG: os.Environ(): " + err.Error()) } } @@ -445,12 +458,13 @@ func (i *InputResolver) getEnvVar(namePattern string) (map[string]string, error) } func (i *InputResolver) resolveEnvVarInputs(inputs []cfg.EnvVarsInputs) (map[string]string, error) { + resolvedEnvVars := map[string]string{} + if len(inputs) == 0 { - return nil, nil + return resolvedEnvVars, nil } i.setEnvVars() - resolvedEnvVars := map[string]string{} for _, e := range inputs { for _, pattern := range e.Names { diff --git a/pkg/baur/inputresolver_test.go b/pkg/baur/inputresolver_test.go index c8e219e2f..395778e2c 100644 --- a/pkg/baur/inputresolver_test.go +++ b/pkg/baur/inputresolver_test.go @@ -978,3 +978,32 @@ func TestFileInSymlinkDir(t *testing.T) { testFn(true) testFn(false) } + +func TestCommandEnvVarsAreTracked(t *testing.T) { + log.RedirectToTestingLog(t) + ir := NewInputResolver(&DummyGitUntrackedFilesResolver{}, ".", true) + inputs, err := ir.Resolve(context.Background(), &Task{ + EnvironmentVariables: []string{"V1=A", "V2=B"}, + UnresolvedInputs: &cfg.Input{}, + }) + require.NoError(t, err) + require.Len(t, inputs, 2) + require.IsType(t, &InputEnvVar{}, inputs[0]) + require.IsType(t, &InputEnvVar{}, inputs[1]) + e1 := inputs[0].(*InputEnvVar) + e2 := inputs[1].(*InputEnvVar) + var f1, f2 *InputEnvVar + + if e1.Name() == "V1" { + f1 = e1 + f2 = e2 + } else { + f1 = e2 + f2 = e1 + } + + assert.Equal(t, "V1", f1.Name()) + assert.Equal(t, "A", f1.value) + assert.Equal(t, "V2", f2.Name()) + assert.Equal(t, "B", f2.value) +} diff --git a/pkg/baur/task.go b/pkg/baur/task.go index 8730259bf..02f6e6e9a 100644 --- a/pkg/baur/task.go +++ b/pkg/baur/task.go @@ -16,24 +16,26 @@ type Task struct { AppName string - Name string - Command []string - UnresolvedInputs *cfg.Input - Outputs *cfg.Output - CfgFilepaths []string + Name string + Command []string + EnvironmentVariables []string + UnresolvedInputs *cfg.Input + Outputs *cfg.Output + CfgFilepaths []string } // NewTask returns a new Task. func NewTask(cfg *cfg.Task, appName, repositoryRootdir, workingDir string) *Task { return &Task{ - RepositoryRoot: repositoryRootdir, - Directory: workingDir, - Outputs: &cfg.Output, - CfgFilepaths: cfg.Filepaths(), - Command: cfg.Command, - Name: cfg.Name, - AppName: appName, - UnresolvedInputs: &cfg.Input, + RepositoryRoot: repositoryRootdir, + Directory: workingDir, + Outputs: &cfg.Output, + CfgFilepaths: cfg.Filepaths(), + Command: cfg.Command, + EnvironmentVariables: cfg.Environment.Variables, + Name: cfg.Name, + AppName: appName, + UnresolvedInputs: &cfg.Input, } } diff --git a/pkg/baur/taskrunner.go b/pkg/baur/taskrunner.go index 6db658b29..3716de983 100644 --- a/pkg/baur/taskrunner.go +++ b/pkg/baur/taskrunner.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "os" "sync/atomic" "time" @@ -62,6 +63,7 @@ func (t *TaskRunner) Run(task *Task) (*RunResult, error) { Directory(task.Directory). LogPrefix(color.YellowString(fmt.Sprintf("%s: ", task))). LogFn(t.LogFn). + Env(append(os.Environ(), task.EnvironmentVariables...)). Run(context.TODO()) if err != nil { return nil, err diff --git a/pkg/baur/taskrunner_test.go b/pkg/baur/taskrunner_test.go index 6bb36e753..0f5e56287 100644 --- a/pkg/baur/taskrunner_test.go +++ b/pkg/baur/taskrunner_test.go @@ -15,3 +15,14 @@ func TestRunningTaskFailsWhenGitWorktreeIsDirty(t *testing.T) { var eu *ErrUntrackedGitFilesExist require.ErrorAs(t, err, &eu) } + +func TestEnvVarIsSet(t *testing.T) { + tr := NewTaskRunner() + res, err := tr.Run(&Task{ + Command: []string{"sh", "-c", `env; if [ "$EV" = "VAL UE" ] && [ "$NOT_EXIST_EV" = "" ]; then exit 0; else exit 1; fi`}, + EnvironmentVariables: []string{"EV=VAL UE"}, + }) + require.NoError(t, err) + require.NoError(t, res.ExpectSuccess()) + +} diff --git a/pkg/cfg/app.go b/pkg/cfg/app.go index 5833c7405..3222bff26 100644 --- a/pkg/cfg/app.go +++ b/pkg/cfg/app.go @@ -26,6 +26,9 @@ func ExampleApp(name string) *App { { Name: "build", Command: []string{"make", "dist"}, + Environment: Environment{ + Variables: []string{"DEBUG=true"}, + }, Input: Input{ Files: []FileInputs{ { diff --git a/pkg/cfg/app_test.go b/pkg/cfg/app_test.go index a31e6d58f..125ffdd39 100644 --- a/pkg/cfg/app_test.go +++ b/pkg/cfg/app_test.go @@ -5,6 +5,8 @@ import ( "path/filepath" "testing" + "github.com/simplesurance/baur/v3/internal/prettyprint" + "github.com/stretchr/testify/require" ) @@ -33,7 +35,7 @@ func Test_ExampleApp_WrittenAndReadCfgIsValid(t *testing.T) { } if err := rRead.Validate(); err != nil { - t.Errorf("validating conf from file failed: %s\nFile Content: %+v", err, rRead) + t.Errorf("validating conf after writing and reading it again from file failed: %s\nFile Content: %+v", err, prettyprint.AsString(rRead)) } } diff --git a/pkg/cfg/environment.go b/pkg/cfg/environment.go new file mode 100644 index 000000000..c224333e0 --- /dev/null +++ b/pkg/cfg/environment.go @@ -0,0 +1,20 @@ +package cfg + +import ( + "fmt" + "strings" +) + +type Environment struct { + Variables []string `toml:"variables" comment:"environment variables, in the format KEY=VALUE, that are set when the command is executed.\n The variables and their values are tracked automatically as inputs."` +} + +func (e *Environment) validate() error { + for _, v := range e.Variables { + if !strings.ContainsRune(v, '=') { + return newFieldError(fmt.Sprintf("'=' missing in %q, environment variables must be defined in the format NAME=VALUE", v), "variables") + } + } + + return nil +} diff --git a/pkg/cfg/includedb_test.go b/pkg/cfg/includedb_test.go index f7f0921c8..d1d3b4f62 100644 --- a/pkg/cfg/includedb_test.go +++ b/pkg/cfg/includedb_test.go @@ -129,6 +129,9 @@ func TestLoadTaskIncludeWithIncludesInSameFile(t *testing.T) { Name: "build", Command: []string{"make"}, Includes: []string{inclFilePath + "#inputs", inclFilePath + "#outputs"}, + Environment: Environment{ + Variables: []string{"K1=V1"}, + }, }, } @@ -151,6 +154,7 @@ func TestLoadTaskIncludeWithIncludesInSameFile(t *testing.T) { assert.Equal(t, include.Task[0].IncludeID, loadedIncl.IncludeID) assert.Equal(t, include.Task[0].Command, loadedIncl.Command) assert.Equal(t, include.Task[0].Includes, loadedIncl.Includes) + assert.Equal(t, include.Task[0].Environment, loadedIncl.Environment) assert.ElementsMatch(t, loadedIncl.Input.Files, include.Input[0].Files) assert.Equal(t, include.Input[0].GolangSources, loadedIncl.Input.GolangSources) diff --git a/pkg/cfg/task.go b/pkg/cfg/task.go index 6635e3c2d..6dd4bd5b0 100644 --- a/pkg/cfg/task.go +++ b/pkg/cfg/task.go @@ -2,11 +2,12 @@ package cfg // Task is a task section type Task struct { - Name string `toml:"name" comment:"Task name"` - Command []string `toml:"command" comment:"Command to execute.\n The first element is the command, the following its arguments."` - Includes []string `toml:"includes" comment:"Input or Output includes that the task inherits.\n Includes are specified in the format FILEPATH#INCLUDE_ID>.\n Paths are relative to the application directory."` - Input Input `toml:"Input" comment:"Inputs are tracked, when they change the task is rerun."` - Output Output `toml:"Output" comment:"Artifacts produced by the Task.command and their upload destinations."` + Name string `toml:"name" comment:"Task name"` + Command []string `toml:"command" comment:"Command to execute.\n The first element is the command, the following its arguments."` + Includes []string `toml:"includes" comment:"Input or Output includes that the task inherits.\n Includes are specified in the format FILEPATH#INCLUDE_ID>.\n Paths are relative to the application directory."` + Environment Environment `toml:"Environment"` + Input Input `toml:"Input" comment:"Inputs are tracked, when they change the task is rerun."` + Output Output `toml:"Output" comment:"Artifacts produced by the Task.command and their upload destinations."` // multiple include sections of the same file can be included, use a map // instead of a slice to act as a Set datastructure @@ -36,6 +37,11 @@ func (t *Task) Filepaths() []string { func (t *Task) command() []string { return t.Command } + +func (t *Task) environment() *Environment { + return &t.Environment +} + func (t *Task) name() string { return t.Name } diff --git a/pkg/cfg/taskdef.go b/pkg/cfg/taskdef.go index 152ec73ac..5d85e227b 100644 --- a/pkg/cfg/taskdef.go +++ b/pkg/cfg/taskdef.go @@ -10,6 +10,7 @@ type taskDef interface { command() []string includes() *[]string input() *Input + environment() *Environment name() string output() *Output addCfgFilepath(path string) @@ -66,6 +67,10 @@ func taskValidate(t taskDef) error { return newFieldError("dots are not allowed in task names", "name") } + if err := t.environment().validate(); err != nil { + return fieldErrorWrap(err, "Environment") + } + if err := validateIncludes(*t.includes()); err != nil { return fieldErrorWrap(err, "includes") } diff --git a/pkg/cfg/taskinclude.go b/pkg/cfg/taskinclude.go index ef9fd221c..5725cd133 100644 --- a/pkg/cfg/taskinclude.go +++ b/pkg/cfg/taskinclude.go @@ -8,11 +8,12 @@ import ( type TaskInclude struct { IncludeID string `toml:"include_id" comment:"identifier of the include"` - Name string `toml:"name" comment:"Task name"` - Command []string `toml:"command" comment:"Command to execute. The first element is the command, the following its arguments.\n If the command element contains no path seperators, its path is looked up via the $PATH environment variable."` - Includes []string `toml:"includes" comment:"Input or Output includes that the task inherits.\n Includes are specified in the format #.\n Paths are relative to the include file location."` - Input Input `toml:"Input" comment:"Specification of task inputs like source files, Makefiles, etc"` - Output Output `toml:"Output" comment:"Specification of task outputs produced by the Task.command"` + Name string `toml:"name" comment:"Task name"` + Command []string `toml:"command" comment:"Command to execute. The first element is the command, the following its arguments.\n If the command element contains no path seperators, its path is looked up via the $PATH environment variable."` + Environment Environment `toml:"Environment"` + Includes []string `toml:"includes" comment:"Input or Output includes that the task inherits.\n Includes are specified in the format #.\n Paths are relative to the include file location."` + Input Input `toml:"Input" comment:"Specification of task inputs like source files, Makefiles, etc"` + Output Output `toml:"Output" comment:"Specification of task outputs produced by the Task.command"` cfgFiles map[string]struct{} } @@ -25,6 +26,10 @@ func (t *TaskInclude) command() []string { return t.Command } +func (t *TaskInclude) environment() *Environment { + return &t.Environment +} + func (t *TaskInclude) name() string { return t.Name } @@ -68,6 +73,7 @@ func (t *TaskInclude) toTask() *Task { deepcopy.MustCopy(t.Input, &result.Input) deepcopy.MustCopy(t.Output, &result.Output) + deepcopy.MustCopy(t.Environment, &result.Environment) return &result }