diff --git a/cli/run.go b/cli/run.go index 543ccee5d..7ea90ec19 100644 --- a/cli/run.go +++ b/cli/run.go @@ -307,7 +307,7 @@ func (r WorkflowRunner) runWorkflows(tracker analytics.Tracker) (models.BuildRun ProjectType: r.config.Config.ProjectType, } - plan, err := createWorkflowRunPlan(r.config.Modes, r.config.Workflow, r.config.Config.Workflows, func() string { return uuid.Must(uuid.NewV4()).String() }) + plan, err := createWorkflowRunPlan(r.config.Modes, r.config.Workflow, r.config.Config.Workflows, r.config.Config.StepBundles, func() string { return uuid.Must(uuid.NewV4()).String() }) if err != nil { return models.BuildRunResultsModel{}, fmt.Errorf("failed to create workflow execution plan: %w", err) } @@ -495,22 +495,39 @@ func registerRunModes(modes models.WorkflowRunModes) error { return nil } -func createWorkflowRunPlan(modes models.WorkflowRunModes, targetWorkflow string, workflows map[string]models.WorkflowModel, uuidProvider func() string) (models.WorkflowRunPlan, error) { +func createWorkflowRunPlan(modes models.WorkflowRunModes, targetWorkflow string, workflows map[string]models.WorkflowModel, stepBundles map[string]models.StepBundleModel, uuidProvider func() string) (models.WorkflowRunPlan, error) { var executionPlan []models.WorkflowExecutionPlan workflowList := walkWorkflows(targetWorkflow, workflows, nil) for _, workflowID := range workflowList { workflow := workflows[workflowID] - var stepPlan []models.StepExecutionPlan + var stepPlans []models.StepExecutionPlan for _, stepListItem := range workflow.Steps { - key, step, with, err := stepListItem.GetStepListItemKeyAndValue() + key, t, err := stepListItem.GetKeyAndType() if err != nil { return models.WorkflowRunPlan{}, err } - if key == models.StepListItemWithKey { + if t == models.StepListItemTypeStep { + step, err := stepListItem.GetStep() + if err != nil { + return models.WorkflowRunPlan{}, err + } + + stepID := key + stepPlans = append(stepPlans, models.StepExecutionPlan{ + UUID: uuidProvider(), + StepID: stepID, + Step: *step, + }) + } else if t == models.StepListItemTypeWith { + with, err := stepListItem.GetWith() + if err != nil { + return models.WorkflowRunPlan{}, err + } + groupID := uuidProvider() for _, stepListStepItem := range with.Steps { @@ -519,22 +536,49 @@ func createWorkflowRunPlan(modes models.WorkflowRunModes, targetWorkflow string, return models.WorkflowRunPlan{}, err } - stepPlan = append(stepPlan, models.StepExecutionPlan{ - UUID: uuidProvider(), - StepID: stepID, - Step: step, - GroupID: groupID, - ContainerID: with.ContainerID, - ServiceIDs: with.ServiceIDs, + stepPlans = append(stepPlans, models.StepExecutionPlan{ + UUID: uuidProvider(), + StepID: stepID, + Step: step, + WithGroupUUID: groupID, + ContainerID: with.ContainerID, + ServiceIDs: with.ServiceIDs, }) } - } else { - stepID := key - stepPlan = append(stepPlan, models.StepExecutionPlan{ - UUID: uuidProvider(), - StepID: stepID, - Step: step, - }) + } else if t == models.StepListItemTypeBundle { + bundleID := key + bundleOverride, err := stepListItem.GetBundle() + if err != nil { + return models.WorkflowRunPlan{}, err + } + + bundleDefinition, ok := stepBundles[bundleID] + if !ok { + return models.WorkflowRunPlan{}, fmt.Errorf("referenced step bundle not defined: %s", bundleID) + } + + bundleEnvs := append(bundleDefinition.Environments, bundleOverride.Environments...) + bundleUUID := uuidProvider() + + for idx, stepListStepItem := range bundleDefinition.Steps { + stepID, step, err := stepListStepItem.GetStepIDAndStep() + if err != nil { + return models.WorkflowRunPlan{}, err + } + + stepPlan := models.StepExecutionPlan{ + UUID: uuidProvider(), + StepID: stepID, + Step: step, + StepBundleUUID: bundleUUID, + } + + if idx == 0 { + stepPlan.StepBundleEnvs = bundleEnvs + } + + stepPlans = append(stepPlans, stepPlan) + } } } @@ -546,7 +590,7 @@ func createWorkflowRunPlan(modes models.WorkflowRunModes, targetWorkflow string, executionPlan = append(executionPlan, models.WorkflowExecutionPlan{ UUID: uuidProvider(), WorkflowID: workflowID, - Steps: stepPlan, + Steps: stepPlans, WorkflowTitle: workflowTitle, IsSteplibOfflineMode: modes.IsSteplibOfflineMode, }) diff --git a/cli/run_test.go b/cli/run_test.go index d4163b82b..9b7b89a35 100644 --- a/cli/run_test.go +++ b/cli/run_test.go @@ -478,17 +478,17 @@ workflows: } func TestEnvOrders(t *testing.T) { - t.Log("Only secret env - secret env should be use") - { - inventoryStr := ` + tests := []struct { + name string + secrets string + config string + }{ + { + name: "Only secret env - secret env should be use", + secrets: ` envs: -- ENV_ORDER_TEST: "should be the 1." -` - - inventory, err := bitrise.InventoryModelFromYAMLBytes([]byte(inventoryStr)) - require.NoError(t, err) - - configStr := ` +- ENV_ORDER_TEST: "should be the 1."`, + config: ` format_version: 1.3.0 default_step_lib_source: "https://github.com/bitrise-io/bitrise-steplib.git" @@ -503,32 +503,14 @@ workflows: echo "ENV_ORDER_TEST: $ENV_ORDER_TEST" if [[ "$ENV_ORDER_TEST" != "should be the 1." ]] ; then exit 1 - fi -` - - config, warnings, err := bitrise.ConfigModelFromYAMLBytes([]byte(configStr)) - require.NoError(t, err) - require.Equal(t, 0, len(warnings)) - - require.NoError(t, configs.InitPaths()) - - runConfig := RunConfig{Config: config, Workflow: "test", Secrets: inventory.Envs} - runner := NewWorkflowRunner(runConfig, nil) - _, err = runner.runWorkflows(noOpTracker{}) - require.NoError(t, err) - } - - t.Log("Secret env & app env also defined - app env should be use") - { - inventoryStr := ` + fi`, + }, + { + name: "Secret env & app env also defined - app env should be use", + secrets: ` envs: -- ENV_ORDER_TEST: "should be the 1." -` - - inventory, err := bitrise.InventoryModelFromYAMLBytes([]byte(inventoryStr)) - require.NoError(t, err) - - configStr := ` +- ENV_ORDER_TEST: "should be the 1."`, + config: ` format_version: 1.3.0 default_step_lib_source: "https://github.com/bitrise-io/bitrise-steplib.git" @@ -547,33 +529,14 @@ workflows: echo "ENV_ORDER_TEST: $ENV_ORDER_TEST" if [[ "$ENV_ORDER_TEST" != "should be the 2." ]] ; then exit 1 - fi - -` - - config, warnings, err := bitrise.ConfigModelFromYAMLBytes([]byte(configStr)) - require.NoError(t, err) - require.Equal(t, 0, len(warnings)) - - require.NoError(t, configs.InitPaths()) - - runConfig := RunConfig{Config: config, Workflow: "test", Secrets: inventory.Envs} - runner := NewWorkflowRunner(runConfig, nil) - _, err = runner.runWorkflows(noOpTracker{}) - require.NoError(t, err) - } - - t.Log("Secret env & app env && workflow env also defined - workflow env should be use") - { - inventoryStr := ` + fi`, + }, + { + name: "Secret env & app env && workflow env also defined - workflow env should be use", + secrets: ` envs: -- ENV_ORDER_TEST: "should be the 1." -` - - inventory, err := bitrise.InventoryModelFromYAMLBytes([]byte(inventoryStr)) - require.NoError(t, err) - - configStr := ` +- ENV_ORDER_TEST: "should be the 1."`, + config: ` format_version: 1.3.0 default_step_lib_source: "https://github.com/bitrise-io/bitrise-steplib.git" @@ -594,39 +557,105 @@ workflows: echo "ENV_ORDER_TEST: $ENV_ORDER_TEST" if [[ "$ENV_ORDER_TEST" != "should be the 3." ]] ; then exit 1 - fi -` - - config, warnings, err := bitrise.ConfigModelFromYAMLBytes([]byte(configStr)) - require.NoError(t, err) - require.Equal(t, 0, len(warnings)) - - require.NoError(t, configs.InitPaths()) + fi`, + }, + { + name: "Secret env & app env && workflow env && step output env also defined - step output env should be use", + secrets: ` +envs: +- ENV_ORDER_TEST: "should be the 1."`, + config: ` +format_version: 1.3.0 +default_step_lib_source: "https://github.com/bitrise-io/bitrise-steplib.git" - runConfig := RunConfig{Config: config, Workflow: "test", Secrets: inventory.Envs} - runner := NewWorkflowRunner(runConfig, nil) - _, err = runner.runWorkflows(noOpTracker{}) - require.NoError(t, err) - } +app: + envs: + - ENV_ORDER_TEST: "should be the 2." - t.Log("Secret env & app env && workflow env && step output env also defined - step output env should be use") - { - inventoryStr := ` +workflows: + test: + envs: + - ENV_ORDER_TEST: "should be the 3." + steps: + - script: + inputs: + - content: envman add --key ENV_ORDER_TEST --value "should be the 4." + - script: + inputs: + - content: | + #!/bin/bash + set -v + echo "ENV_ORDER_TEST: $ENV_ORDER_TEST" + if [[ "$ENV_ORDER_TEST" != "should be the 4." ]] ; then + exit 1 + fi`, + }, + { + name: "Step Bundle definition's env var overrides secrets, app, workflow and step output env vars with the same env key", + secrets: ` envs: -- ENV_ORDER_TEST: "should be the 1." -` +- ENV_ORDER_TEST: "should be the 1."`, + config: ` +format_version: "15" +default_step_lib_source: "https://github.com/bitrise-io/bitrise-steplib.git" - inventory, err := bitrise.InventoryModelFromYAMLBytes([]byte(inventoryStr)) - require.NoError(t, err) +app: + envs: + - ENV_ORDER_TEST: "should be the 2." - configStr := ` -format_version: 1.3.0 +step_bundles: + test-bundle: + envs: + - ENV_ORDER_TEST: "should be the 5." + steps: + - script: + inputs: + - content: | + #!/bin/bash + set -v + echo "ENV_ORDER_TEST: $ENV_ORDER_TEST" + if [[ "$ENV_ORDER_TEST" != "should be the 5." ]] ; then + exit 1 + fi + +workflows: + test: + envs: + - ENV_ORDER_TEST: "should be the 3." + steps: + - script: + inputs: + - content: envman add --key ENV_ORDER_TEST --value "should be the 4." + - bundle::test-bundle: {}`, + }, + { + name: "Step Bundle list item's env var overrides secrets, app, workflow, step output and step bundle definition env vars with the same env key", + secrets: ` +envs: +- ENV_ORDER_TEST: "should be the 1."`, + config: ` +format_version: "15" default_step_lib_source: "https://github.com/bitrise-io/bitrise-steplib.git" app: envs: - ENV_ORDER_TEST: "should be the 2." +step_bundles: + test-bundle: + envs: + - ENV_ORDER_TEST: "should be the 5." + steps: + - script: + inputs: + - content: | + #!/bin/bash + set -v + echo "ENV_ORDER_TEST: $ENV_ORDER_TEST" + if [[ "$ENV_ORDER_TEST" != "should be the 6." ]] ; then + exit 1 + fi + workflows: test: envs: @@ -635,27 +664,119 @@ workflows: - script: inputs: - content: envman add --key ENV_ORDER_TEST --value "should be the 4." + - bundle::test-bundle: + envs: + - ENV_ORDER_TEST: "should be the 6."`, + }, + { + name: "Step Bundle input envs are not shared with the workflow", + config: ` +format_version: "15" +default_step_lib_source: "https://github.com/bitrise-io/bitrise-steplib.git" + +step_bundles: + test-bundle: + envs: + - ENV_ORDER_TEST: "should be the 2." + steps: - script: inputs: - content: | #!/bin/bash set -v echo "ENV_ORDER_TEST: $ENV_ORDER_TEST" - if [[ "$ENV_ORDER_TEST" != "should be the 4." ]] ; then + +workflows: + test: + envs: + - ENV_ORDER_TEST: "should be the 1." + steps: + - bundle::test-bundle: + envs: + - ENV_ORDER_TEST: "should be the 3." + - script: + inputs: + - content: | + #!/bin/bash + set -v + echo "ENV_ORDER_TEST: $ENV_ORDER_TEST" + if [[ "$ENV_ORDER_TEST" != "should be the 1." ]] ; then + exit 1 + fi`, + }, + { + name: "Step Bundle output envs are shared with the workflow", + config: ` +format_version: "15" +default_step_lib_source: "https://github.com/bitrise-io/bitrise-steplib.git" + +step_bundles: + test-bundle-1: + steps: + - script: + inputs: + - content: | + #!/bin/bash + set -v + envman add --key ENV_ORDER_TEST --value "should be the 2." + + test-bundle-2: + steps: + - script: + inputs: + - content: | + #!/bin/bash + set -v + echo "ENV_ORDER_TEST: $ENV_ORDER_TEST" + if [[ "$ENV_ORDER_TEST" != "should be the 2." ]] ; then exit 1 fi -` + envman add --key ENV_ORDER_TEST --value "should be the 3." - config, warnings, err := bitrise.ConfigModelFromYAMLBytes([]byte(configStr)) - require.NoError(t, err) - require.Equal(t, 0, len(warnings)) +workflows: + test: + envs: + - ENV_ORDER_TEST: "should be the 1." + steps: + - bundle::test-bundle-1: {} + - script: + inputs: + - content: | + #!/bin/bash + set -v + echo "ENV_ORDER_TEST: $ENV_ORDER_TEST" + if [[ "$ENV_ORDER_TEST" != "should be the 2." ]] ; then + exit 1 + fi + - bundle::test-bundle-2: {} + - script: + inputs: + - content: | + #!/bin/bash + set -v + echo "ENV_ORDER_TEST: $ENV_ORDER_TEST" + if [[ "$ENV_ORDER_TEST" != "should be the 3." ]] ; then + exit 1 + fi`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + inventory, err := bitrise.InventoryModelFromYAMLBytes([]byte(tt.secrets)) + require.NoError(t, err) - require.NoError(t, configs.InitPaths()) + config, warnings, err := bitrise.ConfigModelFromYAMLBytes([]byte(tt.config)) + require.NoError(t, err) + require.Equal(t, 0, len(warnings)) - runConfig := RunConfig{Config: config, Workflow: "test", Secrets: inventory.Envs} - runner := NewWorkflowRunner(runConfig, nil) - _, err = runner.runWorkflows(noOpTracker{}) - require.NoError(t, err) + require.NoError(t, configs.InitPaths()) + + runConfig := RunConfig{Config: config, Workflow: "test", Secrets: inventory.Envs} + runner := NewWorkflowRunner(runConfig, nil) + res, err := runner.runWorkflows(noOpTracker{}) + require.NoError(t, err) + require.False(t, res.IsBuildFailed()) + }) } } @@ -1143,6 +1264,62 @@ workflows: require.Equal(t, "1", os.Getenv("STEPLIB_BUILD_STATUS")) } +// Checks if BuildStatusEnv is set correctly for Step Bundles +func TestBuildFailedModeForStepBundles(t *testing.T) { + configStr := ` +format_version: "15" +default_step_lib_source: "https://github.com/bitrise-io/bitrise-steplib.git" + +step_bundles: + test-bundle-1: + steps: + - script: + inputs: + - content: exit 0 + + test-bundle-2: + steps: + - script: + is_always_run: true + inputs: + - content: exit 1 + +workflows: + test: + steps: + - bundle::test-bundle-1: {} + - script: + run_if: false + inputs: + - content: exit 1 + - bundle::test-bundle-2: {} + - script: + inputs: + - content: exit 0` + + config, warnings, err := bitrise.ConfigModelFromYAMLBytes([]byte(configStr)) + require.NoError(t, err) + require.Equal(t, 0, len(warnings)) + require.NoError(t, configs.InitPaths()) + + runConfig := RunConfig{Config: config, Workflow: "test"} + runner := NewWorkflowRunner(runConfig, nil) + buildRunResults, err := runner.runWorkflows(noOpTracker{}) + require.NoError(t, err) + + require.Equal(t, 1, len(buildRunResults.SuccessSteps)) + require.Equal(t, 2, len(buildRunResults.SkippedSteps)) + require.Equal(t, 1, len(buildRunResults.FailedSteps)) + + require.Equal(t, 0, buildRunResults.SuccessSteps[0].Idx) + require.Equal(t, 1, buildRunResults.SkippedSteps[0].Idx) + require.Equal(t, 2, buildRunResults.FailedSteps[0].Idx) + require.Equal(t, 3, buildRunResults.SkippedSteps[1].Idx) + + require.Equal(t, "1", os.Getenv("BITRISE_BUILD_STATUS")) + require.Equal(t, "1", os.Getenv("STEPLIB_BUILD_STATUS")) +} + // Trivial test for workflow environment handling, before workflows env should be visible in target and after workflow func TestWorkflowEnvironments(t *testing.T) { configStr := ` diff --git a/cli/run_util.go b/cli/run_util.go index f8ee78a7a..52f15bcb7 100644 --- a/cli/run_util.go +++ b/cli/run_util.go @@ -83,16 +83,39 @@ func (r WorkflowRunner) activateAndRunSteps( runResultCollector := newBuildRunResultCollector(r.logger, tracker) currentStepGroupID := "" + // Global variables for restricting Step Bundle's environment variables for the given Step Bundle + currentStepBundleUUID := "" + // TODO: add the last step bundle's envs to environments + var currentStepBundleEnvVars []envmanModels.EnvironmentItemModel + // ------------------------------------------ // Main - Preparing & running the steps for idx, stepPlan := range plan.Steps { - if stepPlan.GroupID != currentStepGroupID { - if stepPlan.GroupID != "" { + if stepPlan.WithGroupUUID != currentStepGroupID { + if stepPlan.WithGroupUUID != "" { if len(stepPlan.ContainerID) > 0 || len(stepPlan.ServiceIDs) > 0 { - r.startContainersForStepGroup(stepPlan.ContainerID, stepPlan.ServiceIDs, *environments, stepPlan.GroupID, plan.WorkflowTitle) + r.startContainersForStepGroup(stepPlan.ContainerID, stepPlan.ServiceIDs, *environments, stepPlan.WithGroupUUID, plan.WorkflowTitle) } } - currentStepGroupID = stepPlan.GroupID + + currentStepGroupID = stepPlan.WithGroupUUID + } + + workflowEnvironments := append([]envmanModels.EnvironmentItemModel{}, *environments...) + + if stepPlan.StepBundleUUID != currentStepBundleUUID { + if stepPlan.StepBundleUUID != "" { + currentStepBundleEnvVars = append(workflowEnvironments, stepPlan.StepBundleEnvs...) + } + + currentStepBundleUUID = stepPlan.StepBundleUUID + } + + var envsForStepRun []envmanModels.EnvironmentItemModel + if currentStepBundleUUID != "" { + envsForStepRun = currentStepBundleEnvVars + } else { + envsForStepRun = workflowEnvironments } stepStartTime := time.Now() @@ -106,23 +129,26 @@ func (r WorkflowRunner) activateAndRunSteps( defaultStepLibSource, stepPlan.UUID, tracker, - *environments, + envsForStepRun, secrets, buildRunResults, plan.IsSteplibOfflineMode, stepPlan.ContainerID, - stepPlan.GroupID, + stepPlan.WithGroupUUID, stepStartTime, stepStartedProperties, ) *environments = append(*environments, result.OutputEnvironments...) + if currentStepBundleUUID != "" { + currentStepBundleEnvVars = append(currentStepBundleEnvVars, result.OutputEnvironments...) + } isLastStepInWorkflow := idx == len(plan.Steps)-1 // Shut down containers if the step is in a 'With' group, and it's the last step in the group if currentStepGroupID != "" { - doesStepGroupChange := idx < len(plan.Steps)-1 && currentStepGroupID != plan.Steps[idx+1].GroupID + doesStepGroupChange := idx < len(plan.Steps)-1 && currentStepGroupID != plan.Steps[idx+1].WithGroupUUID if isLastStepInWorkflow || doesStepGroupChange { r.stopContainersForStepGroup(currentStepGroupID, plan.WorkflowTitle) } diff --git a/models/models.go b/models/models.go index 9ee0c1f1e..ad6bb0756 100644 --- a/models/models.go +++ b/models/models.go @@ -10,10 +10,22 @@ import ( ) const ( - FormatVersion = "15" - StepListItemWithKey = "with" + FormatVersion = "16" + StepListItemWithKey = "with" + StepListItemStepBundleKeyPrefix = "bundle::" ) +type StepBundleModel struct { + Environments []envmanModels.EnvironmentItemModel `json:"envs,omitempty" yaml:"envs,omitempty"` + Steps []StepListStepItemModel `json:"steps,omitempty" yaml:"steps,omitempty"` +} + +type StepBundleListItemModel struct { + Environments []envmanModels.EnvironmentItemModel `json:"envs,omitempty" yaml:"envs,omitempty"` +} + +type StepListStepBundleItemModel map[string]StepBundleListItemModel + type WithModel struct { ContainerID string `json:"container,omitempty" yaml:"container,omitempty"` ServiceIDs []string `json:"services,omitempty" yaml:"services,omitempty"` @@ -94,14 +106,15 @@ type BitriseDataModel struct { Summary string `json:"summary,omitempty" yaml:"summary,omitempty"` Description string `json:"description,omitempty" yaml:"description,omitempty"` // - Services map[string]Container `json:"services,omitempty" yaml:"services,omitempty"` - Containers map[string]Container `json:"containers,omitempty" yaml:"containers,omitempty"` - App AppModel `json:"app,omitempty" yaml:"app,omitempty"` - Meta map[string]interface{} `json:"meta,omitempty" yaml:"meta,omitempty"` - TriggerMap TriggerMapModel `json:"trigger_map,omitempty" yaml:"trigger_map,omitempty"` - Pipelines map[string]PipelineModel `json:"pipelines,omitempty" yaml:"pipelines,omitempty"` - Stages map[string]StageModel `json:"stages,omitempty" yaml:"stages,omitempty"` - Workflows map[string]WorkflowModel `json:"workflows,omitempty" yaml:"workflows,omitempty"` + Services map[string]Container `json:"services,omitempty" yaml:"services,omitempty"` + Containers map[string]Container `json:"containers,omitempty" yaml:"containers,omitempty"` + App AppModel `json:"app,omitempty" yaml:"app,omitempty"` + Meta map[string]interface{} `json:"meta,omitempty" yaml:"meta,omitempty"` + TriggerMap TriggerMapModel `json:"trigger_map,omitempty" yaml:"trigger_map,omitempty"` + Pipelines map[string]PipelineModel `json:"pipelines,omitempty" yaml:"pipelines,omitempty"` + Stages map[string]StageModel `json:"stages,omitempty" yaml:"stages,omitempty"` + Workflows map[string]WorkflowModel `json:"workflows,omitempty" yaml:"workflows,omitempty"` + StepBundles map[string]StepBundleModel `json:"step_bundles,omitempty" yaml:"step_bundles,omitempty"` } type BuildRunStartModel struct { diff --git a/models/models_methods.go b/models/models_methods.go index 8b858374e..f23cb6c49 100644 --- a/models/models_methods.go +++ b/models/models_methods.go @@ -1,6 +1,7 @@ package models import ( + "encoding/json" "errors" "fmt" "regexp" @@ -105,6 +106,33 @@ func (config *BitriseDataModel) getPipelineIDs() []string { // ---------------------------- // --- Normalize +func (bundle *StepBundleListItemModel) Normalize() error { + for _, env := range bundle.Environments { + if err := env.Normalize(); err != nil { + return err + } + } + return nil +} + +func (bundle *StepBundleModel) Normalize() error { + for _, env := range bundle.Environments { + if err := env.Normalize(); err != nil { + return err + } + } + return nil +} + +func (container *Container) Normalize() error { + for _, env := range container.Envs { + if err := env.Normalize(); err != nil { + return err + } + } + return nil +} + func (workflow *WorkflowModel) Normalize() error { for _, env := range workflow.Environments { if err := env.Normalize(); err != nil { @@ -113,15 +141,28 @@ func (workflow *WorkflowModel) Normalize() error { } for _, stepListItem := range workflow.Steps { - key, step, _, err := stepListItem.GetStepListItemKeyAndValue() + _, t, err := stepListItem.GetKeyAndType() if err != nil { return err } - if key != StepListItemWithKey { + if t == StepListItemTypeStep { + step, err := stepListItem.GetStep() + if err != nil { + return err + } + if err := step.Normalize(); err != nil { return err } - stepListItem[key] = step + } else if t == StepListItemTypeBundle { + bundle, err := stepListItem.GetBundle() + if err != nil { + return err + } + + if err := bundle.Normalize(); err != nil { + return err + } } } @@ -139,23 +180,42 @@ func (app *AppModel) Normalize() error { func (config *BitriseDataModel) Normalize() error { if err := config.App.Normalize(); err != nil { - return err + return fmt.Errorf("failed to normalize app: %w", err) } normalizedTriggerMap, err := config.TriggerMap.Normalized() if err != nil { - return err + return fmt.Errorf("failed to normalize trigger_map: %w", err) } config.TriggerMap = normalizedTriggerMap + for _, container := range config.Containers { + if err := container.Normalize(); err != nil { + return fmt.Errorf("failed to normalize container: %w", err) + } + } + + for _, container := range config.Services { + if err := container.Normalize(); err != nil { + return fmt.Errorf("failed to normalize service: %w", err) + } + } + + for _, stepBundle := range config.StepBundles { + if err := stepBundle.Normalize(); err != nil { + return fmt.Errorf("failed to normalize step_bundle: %w", err) + } + } + for _, workflow := range config.Workflows { if err := workflow.Normalize(); err != nil { - return err + return fmt.Errorf("failed to normalize workflow: %w", err) } } + normalizedMeta, err := stepmanModels.JSONMarshallable(config.Meta) if err != nil { - return err + return fmt.Errorf("failed to normalize meta: %w", err) } config.Meta = normalizedMeta @@ -165,6 +225,47 @@ func (config *BitriseDataModel) Normalize() error { // ---------------------------- // --- Validate +func (bundle *StepBundleListItemModel) Validate() error { + for _, env := range bundle.Environments { + if err := env.Validate(); err != nil { + return err + } + } + + return nil +} + +func (bundle *StepBundleModel) Validate() ([]string, error) { + for _, env := range bundle.Environments { + if err := env.Validate(); err != nil { + return nil, err + } + } + var warnings []string + for _, stepListItem := range bundle.Steps { + stepID, step, err := stepListItem.GetStepIDAndStep() + if err != nil { + return warnings, err + } + + warns, err := validateStep(stepID, step) + warnings = append(warnings, warns...) + if err != nil { + return warnings, err + } + } + return warnings, nil +} + +func (container *Container) Validate() error { + for _, env := range container.Envs { + if err := env.Validate(); err != nil { + return err + } + } + return nil +} + func (with WithModel) Validate(workflowID string, containers, services map[string]Container) ([]string, error) { var warnings []string @@ -203,63 +304,13 @@ func (with WithModel) Validate(workflowID string, containers, services map[strin } -func (workflow *WorkflowModel) Validate() ([]string, error) { - var warnings []string - +func (workflow *WorkflowModel) Validate() error { for _, env := range workflow.Environments { if err := env.Validate(); err != nil { - return warnings, err - } - } - - for _, stepListItem := range workflow.Steps { - key, step, _, err := stepListItem.GetStepListItemKeyAndValue() - if err != nil { - return warnings, err - } - - if key != StepListItemWithKey { - stepID := key - warns, err := validateStep(stepID, step) - warnings = append(warnings, warns...) - if err != nil { - return warnings, err - } - - // TODO: Why is this assignment needed? - stepListItem[stepID] = step - } - } - - return warnings, nil -} - -func validateStep(stepID string, step stepmanModels.StepModel) ([]string, error) { - var warnings []string - - if err := stepid.Validate(stepID); err != nil { - return warnings, err - } - - if err := step.ValidateInputAndOutputEnvs(false); err != nil { - return warnings, err - } - - stepInputMap := map[string]bool{} - for _, input := range step.Inputs { - key, _, err := input.GetKeyValuePair() - if err != nil { - return warnings, err - } - - _, found := stepInputMap[key] - if found { - warnings = append(warnings, fmt.Sprintf("invalid step: duplicated input found: (%s)", key)) + return err } - stepInputMap[key] = true } - - return warnings, nil + return nil } func (app *AppModel) Validate() error { @@ -281,8 +332,8 @@ func (config *BitriseDataModel) Validate() ([]string, error) { // trigger map workflows := config.getWorkflowIDs() pipelines := config.getPipelineIDs() - warns, err := config.TriggerMap.Validate(workflows, pipelines) - warnings = append(warnings, warns...) + triggerMapWarnings, err := config.TriggerMap.Validate(workflows, pipelines) + warnings = append(warnings, triggerMapWarnings...) if err != nil { return warnings, err } @@ -295,22 +346,16 @@ func (config *BitriseDataModel) Validate() ([]string, error) { // --- // containers - for containerID, containerDef := range config.Containers { - if containerID == "" { - return nil, fmt.Errorf("service (image: %s) has empty ID defined", containerDef.Image) - } - if strings.TrimSpace(containerDef.Image) == "" { - return warnings, fmt.Errorf("service (%s) has no image defined", containerID) - } + if err := validateContainers(*config); err != nil { + return warnings, err } + // --- - for serviceID, serviceDef := range config.Services { - if serviceID == "" { - return nil, fmt.Errorf("service (image: %s) has empty ID defined", serviceDef.Image) - } - if strings.TrimSpace(serviceDef.Image) == "" { - return warnings, fmt.Errorf("service (%s) has no image defined", serviceID) - } + // step_bundles + stepBundleWarnings, err := validateStepBundles(*config) + warnings = append(warnings, stepBundleWarnings...) + if err != nil { + return warnings, err } // --- @@ -336,23 +381,81 @@ func (config *BitriseDataModel) Validate() ([]string, error) { if err != nil { return warnings, err } + // --- - for workflowID, workflow := range config.Workflows { - for _, stepListItem := range workflow.Steps { - key, _, with, err := stepListItem.GetStepListItemKeyAndValue() - if err != nil { - return warnings, err - } - if key == StepListItemWithKey { - warns, err := with.Validate(workflowID, config.Containers, config.Services) - warnings = append(warnings, warns...) - if err != nil { - return warnings, err - } - } + return warnings, nil +} + +func validateContainers(config BitriseDataModel) error { + for containerID, containerDef := range config.Containers { + if containerID == "" { + return fmt.Errorf("container (image: %s) has empty ID defined", containerDef.Image) + } + if strings.TrimSpace(containerDef.Image) == "" { + return fmt.Errorf("container (%s) has no image defined", containerID) + } + if err := containerDef.Validate(); err != nil { + return fmt.Errorf("container (%s) has config issue: %w", containerID, err) + } + } + + for serviceID, serviceDef := range config.Services { + if serviceID == "" { + return fmt.Errorf("service (image: %s) has empty ID defined", serviceDef.Image) + } + if strings.TrimSpace(serviceDef.Image) == "" { + return fmt.Errorf("service (%s) has no image defined", serviceID) + } + if err := serviceDef.Validate(); err != nil { + return fmt.Errorf("container (%s) has config issue: %w", serviceID, err) + } + } + + return nil +} + +func validateStepBundles(config BitriseDataModel) ([]string, error) { + var warnings []string + + for bundleID, bundle := range config.StepBundles { + if bundleID == "" { + return warnings, fmt.Errorf("step bundle has empty ID defined") + } + + warns, err := bundle.Validate() + warnings = append(warnings, warns...) + if err != nil { + return warnings, fmt.Errorf("step bundle (%s) has config issue: %w", bundleID, err) } } - // --- + + return warnings, nil +} + +func validateStep(stepID string, step stepmanModels.StepModel) ([]string, error) { + var warnings []string + + if err := stepid.Validate(stepID); err != nil { + return warnings, err + } + + if err := step.ValidateInputAndOutputEnvs(false); err != nil { + return warnings, err + } + + stepInputMap := map[string]bool{} + for _, input := range step.Inputs { + key, _, err := input.GetKeyValuePair() + if err != nil { + return warnings, err + } + + _, found := stepInputMap[key] + if found { + warnings = append(warnings, fmt.Sprintf("invalid step: duplicated input found: (%s)", key)) + } + stepInputMap[key] = true + } return warnings, nil } @@ -439,28 +542,75 @@ func isUtilityWorkflow(workflowID string) bool { } func validateWorkflows(config *BitriseDataModel) ([]string, error) { - workflowWarnings := make([]string, 0) - for ID, workflow := range config.Workflows { - idWarning, err := validateID(ID, "workflow") + var warnings []string + + for workflowID, workflow := range config.Workflows { + idWarning, err := validateID(workflowID, "workflow") if idWarning != "" { - workflowWarnings = append(workflowWarnings, idWarning) + warnings = append(warnings, idWarning) } if err != nil { - return workflowWarnings, err + return warnings, err } - warns, err := workflow.Validate() - workflowWarnings = append(workflowWarnings, warns...) - if err != nil { - return workflowWarnings, fmt.Errorf("validation error in workflow: %s: %s", ID, err) + if err := checkWorkflowReferenceCycle(workflowID, workflow, *config, []string{}); err != nil { + return warnings, err } - if err := checkWorkflowReferenceCycle(ID, workflow, *config, []string{}); err != nil { - return workflowWarnings, err + if err := workflow.Validate(); err != nil { + return warnings, fmt.Errorf("validation error in workflow: %s: %s", workflowID, err) + } + + for _, stepListItem := range workflow.Steps { + key, t, err := stepListItem.GetKeyAndType() + if err != nil { + return warnings, err + } + + if t == StepListItemTypeStep { + step, err := stepListItem.GetStep() + if err != nil { + return warnings, err + } + stepID := key + warns, err := validateStep(stepID, *step) + warnings = append(warnings, warns...) + if err != nil { + return warnings, err + } + + // TODO: Why is this assignment needed? + stepListItem[stepID] = step + } else if t == StepListItemTypeWith { + with, err := stepListItem.GetWith() + if err != nil { + return warnings, err + } + + warns, err := with.Validate(workflowID, config.Containers, config.Services) + warnings = append(warnings, warns...) + if err != nil { + return warnings, err + } + } else if t == StepListItemTypeBundle { + bundleID := strings.TrimPrefix(key, StepListItemStepBundleKeyPrefix) + if _, ok := config.StepBundles[bundleID]; !ok { + return warnings, fmt.Errorf("step-bundle (%s) referenced in workflow (%s), but this step-bundle is not defined", bundleID, workflowID) + } + + bundle, err := stepListItem.GetBundle() + if err != nil { + return warnings, err + } + + if err := bundle.Validate(); err != nil { + return warnings, err + } + } } } - return workflowWarnings, nil + return warnings, nil } func validateID(id, modelType string) (string, error) { @@ -890,6 +1040,53 @@ func getStageID(stageListItem StageListItemModel) (string, error) { // ---------------------------- // --- StepIDData +func (stepListItem *StepListItemModel) UnmarshalJSON(b []byte) error { + var raw map[string]interface{} + if err := json.Unmarshal(b, &raw); err != nil { + return err + } + + var key string + for k := range raw { + key = k + break + } + + if key == StepListItemWithKey { + var withItem StepListWithItemModel + if err := json.Unmarshal(b, &withItem); err != nil { + return err + } + + *stepListItem = map[string]interface{}{} + for k, v := range withItem { + (*stepListItem)[k] = v + } + } else if strings.HasPrefix(key, StepListItemStepBundleKeyPrefix) { + var stepBundleItem StepListStepBundleItemModel + if err := json.Unmarshal(b, &stepBundleItem); err != nil { + return err + } + + *stepListItem = map[string]interface{}{} + for k, v := range stepBundleItem { + (*stepListItem)[k] = v + } + } else { + var stepItem StepListStepItemModel + if err := json.Unmarshal(b, &stepItem); err != nil { + return err + } + + *stepListItem = map[string]interface{}{} + for k, v := range stepItem { + (*stepListItem)[k] = v + } + } + + return nil +} + func (stepListItem *StepListItemModel) UnmarshalYAML(unmarshal func(interface{}) error) error { var raw map[string]interface{} if err := unmarshal(&raw); err != nil { @@ -912,6 +1109,16 @@ func (stepListItem *StepListItemModel) UnmarshalYAML(unmarshal func(interface{}) for k, v := range withItem { (*stepListItem)[k] = v } + } else if strings.HasPrefix(key, StepListItemStepBundleKeyPrefix) { + var stepBundleItem StepListStepBundleItemModel + if err := unmarshal(&stepBundleItem); err != nil { + return err + } + + *stepListItem = map[string]interface{}{} + for k, v := range stepBundleItem { + (*stepListItem)[k] = v + } } else { var stepItem StepListStepItemModel if err := unmarshal(&stepItem); err != nil { @@ -951,42 +1158,103 @@ func (stepListStepItem *StepListStepItemModel) GetStepIDAndStep() (string, stepm return stepID, step, nil } -// GetStepListItemKeyAndValue returns the Step List Item key and value. The key is either a Step ID or 'with'. -// If the key is 'with' the returned WithModel is relevant otherwise the StepModel. -func (stepListItem *StepListItemModel) GetStepListItemKeyAndValue() (string, stepmanModels.StepModel, WithModel, error) { +type StepListItemType int + +const ( + StepListItemTypeUnknown StepListItemType = iota + StepListItemTypeStep + StepListItemTypeWith + StepListItemTypeBundle +) + +func (stepListItem *StepListItemModel) GetKeyAndType() (string, StepListItemType, error) { if stepListItem == nil { - return "", stepmanModels.StepModel{}, WithModel{}, nil + return "", StepListItemTypeUnknown, nil } if len(*stepListItem) == 0 { - return "", stepmanModels.StepModel{}, WithModel{}, errors.New("StepListItem does not contain a key-value pair") + return "", StepListItemTypeUnknown, errors.New("StepListItem does not contain a key-value pair") } if len(*stepListItem) > 1 { - return "", stepmanModels.StepModel{}, WithModel{}, errors.New("StepListItem contains more than 1 key-value pair") + return "", StepListItemTypeUnknown, errors.New("StepListItem contains more than 1 key-value pair") } - for key, value := range *stepListItem { - if key == StepListItemWithKey { - with := value.(WithModel) - return key, stepmanModels.StepModel{}, with, nil - } else { - step, ok := value.(stepmanModels.StepModel) - if ok { - return key, step, WithModel{}, nil - } + for key := range *stepListItem { + switch { + case strings.HasPrefix(key, StepListItemStepBundleKeyPrefix): + return strings.TrimPrefix(key, StepListItemStepBundleKeyPrefix), StepListItemTypeBundle, nil + case key == StepListItemWithKey: + return key, StepListItemTypeWith, nil + default: + return key, StepListItemTypeStep, nil + } + } - // StepListItemModel is a map[string]interface{}, when it comes from a JSON/YAML unmarshal - // the StepModel has a pointer type. - stepPtr, ok := value.(*stepmanModels.StepModel) - if ok { - return key, *stepPtr, WithModel{}, nil - } + return "", StepListItemTypeUnknown, nil +} + +func (stepListItem *StepListItemModel) GetBundle() (*StepBundleListItemModel, error) { + if stepListItem == nil { + return nil, fmt.Errorf("empty stepListItem") + } - return key, stepmanModels.StepModel{}, WithModel{}, nil + for _, value := range *stepListItem { + bundle, ok := value.(StepBundleListItemModel) + if ok { + return &bundle, nil } + break } - return "", stepmanModels.StepModel{}, WithModel{}, nil + + return nil, fmt.Errorf("stepListItem is not a StepBundle") +} + +func (stepListItem *StepListItemModel) GetWith() (*WithModel, error) { + if stepListItem == nil { + return nil, fmt.Errorf("empty stepListItem") + } + + for _, value := range *stepListItem { + with, ok := value.(WithModel) + if ok { + return &with, nil + } + break + } + + return nil, fmt.Errorf("stepListItem is not a With") +} + +func (stepListItem *StepListItemModel) GetStep() (*stepmanModels.StepModel, error) { + if stepListItem == nil { + return nil, fmt.Errorf("empty stepListItem") + } + + var stepPtr *stepmanModels.StepModel + for _, value := range *stepListItem { + s, ok := value.(stepmanModels.StepModel) + if ok { + stepPtr = &s + break + } + + // StepListItemModel is a map[string]interface{}, when it comes from a JSON/YAML unmarshal + // the StepModel has a pointer type. + sPtr, ok := value.(*stepmanModels.StepModel) + if ok { + stepPtr = sPtr + break + } + + break + } + + if stepPtr == nil { + return nil, fmt.Errorf("stepListItem is not a Step") + } + + return stepPtr, nil } // ---------------------------- diff --git a/models/models_methods_test.go b/models/models_methods_test.go index bcfe94687..d72a92fad 100644 --- a/models/models_methods_test.go +++ b/models/models_methods_test.go @@ -86,6 +86,8 @@ services: image: postgres:13 envs: - POSTGRES_PASSWORD: password + opts: + is_expand: false ports: - 5435:5432 options: >- @@ -128,6 +130,40 @@ workflows: inputs: - content: bundle exec rspec spec/features/`, }, + { + name: "Step Bundles are normalized", + config: ` +format_version: "15" +default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git +project_type: other + +step_bundles: + print-hello: + envs: + - NAME: World + opts: + is_expand: false + steps: + - script: + inputs: + - content: echo "Hello, $NAME!" + +workflows: + print-hellos: + envs: + - NAME: Bitrise + steps: + - bundle::print-hello: {} + - script: + inputs: + - content: echo "Hello, $NAME!" + - bundle::print-hello: + envs: + - NAME: Universe + opts: + is_expand: false +`, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -539,7 +575,7 @@ default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git containers: "": image: ruby:3.2`), - wantErr: "service (image: ruby:3.2) has empty ID defined", + wantErr: "container (image: ruby:3.2) has empty ID defined", }, { name: "Invalid bitrise.yml: missing container image", @@ -549,7 +585,7 @@ default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git containers: "ruby": image: ""`), - wantErr: "service (ruby) has no image defined", + wantErr: "container (ruby) has no image defined", }, { name: "Invalid bitrise.yml: missing container image (whitespace)", @@ -559,7 +595,7 @@ default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git containers: "ruby": image: " "`), - wantErr: "service (ruby) has no image defined", + wantErr: "container (ruby) has no image defined", }, { name: "Invalid bitrise.yml: non-existing container referenced", @@ -624,6 +660,105 @@ workflows: } } +func TestValidateConfig_StepBundles(t *testing.T) { + tests := []struct { + name string + config BitriseDataModel + wantErr string + }{ + { + name: "Valid bitrise.yml with a step bundle", + config: createConfig(t, ` +format_version: "15" +default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git +project_type: other + +step_bundles: + print-hello: + envs: + - NAME: World + opts: + is_expand: false + steps: + - script: + inputs: + - content: echo "Hello, $NAME!" + +workflows: + print-hellos: + envs: + - NAME: Bitrise + steps: + - bundle::print-hello: {} + - script: + inputs: + - content: echo "Hello, $NAME!" + - bundle::print-hello: + envs: + - NAME: Universe +`), + }, + { + name: "Invalid bitrise.yml: empty step bundle ID", + config: createConfig(t, ` +format_version: "15" +default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git +project_type: other + +step_bundles: + "": + envs: + - NAME: World + steps: + - script: + inputs: + - content: echo "Hello, $NAME!" + +workflows: + print-hellos: + steps: + - bundle::print-hello: {} +`), + wantErr: "step bundle has empty ID defined", + }, + { + name: "Invalid bitrise.yml: non-existing container referenced", + config: createConfig(t, ` +format_version: "15" +default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git +project_type: other + +step_bundles: + print-hello: + envs: + - NAME: World + steps: + - script: + inputs: + - content: echo "Hello, $NAME!" + +workflows: + print-hellos: + steps: + - bundle::non-existing-bundle: {} +`), + wantErr: "step-bundle (non-existing-bundle) referenced in workflow (print-hellos), but this step-bundle is not defined", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + warns, err := tt.config.Validate() + require.Empty(t, warns) + + if tt.wantErr != "" { + require.EqualError(t, err, tt.wantErr) + } else { + require.NoError(t, err) + } + }) + } +} + // Workflow func TestValidateWorkflow(t *testing.T) { t.Log("before-after test") @@ -633,9 +768,7 @@ func TestValidateWorkflow(t *testing.T) { AfterRun: []string{"after1", "after2", "after3"}, } - warnings, err := workflow.Validate() - require.NoError(t, err) - require.Equal(t, 0, len(warnings)) + require.NoError(t, workflow.Validate()) } t.Log("invalid workflow - Invalid env: more than 2 fields") @@ -1133,9 +1266,15 @@ func TestGetStepIDStepDataPair(t *testing.T) { "step1": stepData, } - id, _, _, err := stepListItem.GetStepListItemKeyAndValue() + key, itemType, err := stepListItem.GetKeyAndType() + require.NoError(t, err) + require.Equal(t, StepListItemTypeStep, itemType) + + _, err = stepListItem.GetStep() + require.NoError(t, err) + require.NoError(t, err) - require.Equal(t, "step1", id) + require.Equal(t, "step1", key) } t.Log("invalid steplist item - more than 1 step") @@ -1145,18 +1284,19 @@ func TestGetStepIDStepDataPair(t *testing.T) { "step2": stepData, } - id, _, _, err := stepListItem.GetStepListItemKeyAndValue() + key, itemType, err := stepListItem.GetKeyAndType() require.Error(t, err) - require.Equal(t, "", id) + require.Equal(t, "", key) + require.Equal(t, StepListItemTypeUnknown, itemType) } t.Log("invalid steplist item - no step") { stepListItem := StepListItemModel{} - - id, _, _, err := stepListItem.GetStepListItemKeyAndValue() + key, itemType, err := stepListItem.GetKeyAndType() require.Error(t, err) - require.Equal(t, "", id) + require.Equal(t, "", key) + require.Equal(t, StepListItemTypeUnknown, itemType) } } @@ -1343,7 +1483,11 @@ workflows: } for _, stepListItem := range workflow.Steps { - _, step, _, err := stepListItem.GetStepListItemKeyAndValue() + _, itemType, err := stepListItem.GetKeyAndType() + require.NoError(t, err) + require.Equal(t, StepListItemTypeStep, itemType) + + step, err := stepListItem.GetStep() require.NoError(t, err) require.Nil(t, step.Title) diff --git a/models/workflow_run_plan.go b/models/workflow_run_plan.go index 35a920fd5..2dff2d1f8 100644 --- a/models/workflow_run_plan.go +++ b/models/workflow_run_plan.go @@ -3,6 +3,7 @@ package models import ( "time" + envmanModels "github.com/bitrise-io/envman/models" stepmanModels "github.com/bitrise-io/stepman/models" ) @@ -16,15 +17,20 @@ type WorkflowRunModes struct { IsSteplibOfflineMode bool } -// TODO: dispatch Plans from JSON event logging and actual workflow execution +// TODO: separate Plans from JSON event logging and actual workflow execution + type StepExecutionPlan struct { UUID string `json:"uuid"` StepID string `json:"step_id"` - Step stepmanModels.StepModel `json:"-"` - GroupID string `json:"-"` - ContainerID string `json:"-"` - ServiceIDs []string `json:"-"` + Step stepmanModels.StepModel `json:"-"` + // With (container) group + WithGroupUUID string `json:"-"` + ContainerID string `json:"-"` + ServiceIDs []string `json:"-"` + // Step Bundle group + StepBundleUUID string `json:"-"` + StepBundleEnvs []envmanModels.EnvironmentItemModel `json:"-"` } type WorkflowExecutionPlan struct { diff --git a/version/build.go b/version/build.go index ed5585e32..f6df03bd2 100644 --- a/version/build.go +++ b/version/build.go @@ -1,7 +1,7 @@ package version // VERSION is the main CLI version number. It's defined at build time using -ldflags -var VERSION = "2.16.1" +var VERSION = "2.17.0" // BuildNumber is the CI build number that creates the release. It's defined at build time using -ldflags var BuildNumber = ""