diff --git a/docs/ce/howto/generate-projects.mdx b/docs/ce/howto/generate-projects.mdx index 8fec967f..5bb22a3a 100644 --- a/docs/ce/howto/generate-projects.mdx +++ b/docs/ce/howto/generate-projects.mdx @@ -24,10 +24,19 @@ generate_projects: workflow: default_work_flow - include: "environments/dev/**" workflow: default_work_flow - - include: "environments/qa/**" + - include: "environments/dev/**" + exclude: "environments/dev/local-tests/**" + workspace: "qa" workflow: default_work_flow + include_patterns: ["modules/**"] + exclude_patterns: ["modules/dontincludeme/**"] ``` +Note that there is a distinction between `include_patterns`, `exclude_patterns` and `include`, `exclude`. +`include` and `exclude` are used to decide which project get generated in the first place +whereas `include_patterns` and `exclude_patterns` are used to decide which additional folders should cause the generated project to trigger + + [Demo repo](https://github.com/diggerhq/generate_projects_demo) # Traversing nested directories diff --git a/libs/digger_config/digger_config.go b/libs/digger_config/digger_config.go index 16e8c483..93c34142 100644 --- a/libs/digger_config/digger_config.go +++ b/libs/digger_config/digger_config.go @@ -243,12 +243,12 @@ func HandleYamlProjectGeneration(config *DiggerConfigYaml, terraformDir string, if MatchIncludeExcludePatternsToFile(dir, includePatterns, excludePatterns) { projectName := strings.ReplaceAll(dir, "/", "_") project := ProjectYaml{ - Name: projectName, - Dir: dir, - Workflow: defaultWorkflowName, - Workspace: "default", - AwsRoleToAssume: config.GenerateProjectsConfig.AwsRoleToAssume, - Generated: true, + Name: projectName, + Dir: dir, + Workflow: defaultWorkflowName, + Workspace: "default", + AwsRoleToAssume: config.GenerateProjectsConfig.AwsRoleToAssume, + Generated: true, AwsCognitoOidcConfig: config.GenerateProjectsConfig.AwsCognitoOidcConfig, } config.Projects = append(config.Projects, &project) @@ -272,13 +272,13 @@ func HandleYamlProjectGeneration(config *DiggerConfigYaml, terraformDir string, } tgParsingConfig := TerragruntParsingConfig{ - CreateProjectName: true, - DefaultWorkflow: workflow, - WorkflowFile: b.WorkflowFile, - FilterPath: path.Join(terraformDir, *b.RootDir), - AwsRoleToAssume: b.AwsRoleToAssume, + CreateProjectName: true, + DefaultWorkflow: workflow, + WorkflowFile: b.WorkflowFile, + FilterPath: path.Join(terraformDir, *b.RootDir), + AwsRoleToAssume: b.AwsRoleToAssume, AwsCognitoOidcConfig: b.AwsCognitoOidcConfig, - }; + } err := hydrateDiggerConfigYamlWithTerragrunt(config, tgParsingConfig, terraformDir) if err != nil { @@ -294,19 +294,26 @@ func HandleYamlProjectGeneration(config *DiggerConfigYaml, terraformDir string, workflow = b.Workflow } + workspace := "default" + if b.Workspace != "" { + workspace = b.Workspace + } + for _, dir := range dirs { if MatchIncludeExcludePatternsToFile(dir, includePatterns, excludePatterns) { projectName := strings.ReplaceAll(dir, "/", "_") project := ProjectYaml{ - Name: projectName, - Dir: dir, - Workflow: workflow, - Workspace: "default", - OpenTofu: b.OpenTofu, - AwsRoleToAssume: b.AwsRoleToAssume, - Generated: true, + Name: projectName, + Dir: dir, + Workflow: workflow, + Workspace: workspace, + OpenTofu: b.OpenTofu, + AwsRoleToAssume: b.AwsRoleToAssume, + Generated: true, AwsCognitoOidcConfig: b.AwsCognitoOidcConfig, - WorkflowFile: b.WorkflowFile, + WorkflowFile: b.WorkflowFile, + IncludePatterns: b.IncludePatterns, + ExcludePatterns: b.ExcludePatterns, } config.Projects = append(config.Projects, &project) } @@ -538,15 +545,15 @@ func hydrateDiggerConfigYamlWithTerragrunt(configYaml *DiggerConfigYaml, parsing } configYaml.Projects = append(configYaml.Projects, &ProjectYaml{ - Name: atlantisProject.Name, - Dir: projectDir, - Workspace: atlantisProject.Workspace, - Terragrunt: true, - Workflow: atlantisProject.Workflow, - WorkflowFile: workflowFile, - IncludePatterns: atlantisProject.Autoplan.WhenModified, - Generated: true, - AwsRoleToAssume: parsingConfig.AwsRoleToAssume, + Name: atlantisProject.Name, + Dir: projectDir, + Workspace: atlantisProject.Workspace, + Terragrunt: true, + Workflow: atlantisProject.Workflow, + WorkflowFile: workflowFile, + IncludePatterns: atlantisProject.Autoplan.WhenModified, + Generated: true, + AwsRoleToAssume: parsingConfig.AwsRoleToAssume, AwsCognitoOidcConfig: parsingConfig.AwsCognitoOidcConfig, }) } diff --git a/libs/digger_config/digger_config_test.go b/libs/digger_config/digger_config_test.go index c46be0ec..b564d57f 100644 --- a/libs/digger_config/digger_config_test.go +++ b/libs/digger_config/digger_config_test.go @@ -843,7 +843,10 @@ generate_projects: blocks: - include: dev/* exclude: dev/project + include_patterns: ["modules/**"] + exclude_patterns: ["modules/dontincludeme/**"] workflow: dev_workflow + workspace: devdev - include: prod/* exclude: prod/project workflow: prod_workflow @@ -878,9 +881,19 @@ workflows: assert.Equal(t, "dev_workflow", dg.Projects[0].Workflow) assert.Equal(t, "dev_workflow", dg.Projects[1].Workflow) assert.Equal(t, "prod_workflow", dg.Projects[2].Workflow) + assert.Equal(t, "devdev", dg.Projects[0].Workspace) + assert.Equal(t, "devdev", dg.Projects[1].Workspace) + assert.Equal(t, "default", dg.Projects[2].Workspace) assert.Equal(t, "dev/test1", dg.Projects[0].Dir) assert.Equal(t, "dev/test2", dg.Projects[1].Dir) assert.Equal(t, "prod/one", dg.Projects[2].Dir) + assert.Equal(t, []string{"modules/**"}, dg.Projects[0].IncludePatterns) + assert.Equal(t, []string{"modules/**"}, dg.Projects[1].IncludePatterns) + assert.Nil(t, dg.Projects[2].IncludePatterns) + assert.Equal(t, []string{"modules/dontincludeme/**"}, dg.Projects[0].ExcludePatterns) + assert.Equal(t, []string{"modules/dontincludeme/**"}, dg.Projects[1].ExcludePatterns) + assert.Nil(t, dg.Projects[2].ExcludePatterns) + assert.Equal(t, 3, len(dg.Projects)) } @@ -1221,4 +1234,4 @@ projects: assert.Equal(t, dg.Projects[0].AwsCognitoOidcConfig.CognitoPoolId, "us-east-1:00000000-0000-0000-0000-000000000000") assert.Equal(t, dg.Projects[0].AwsCognitoOidcConfig.AwsAccountId, "000000000000") -} \ No newline at end of file +} diff --git a/libs/digger_config/utils_test.go b/libs/digger_config/utils_test.go index c75b4474..7c5d857d 100644 --- a/libs/digger_config/utils_test.go +++ b/libs/digger_config/utils_test.go @@ -19,6 +19,13 @@ func TestMatchIncludeExcludePatternsToFile(t *testing.T) { result = MatchIncludeExcludePatternsToFile("projects/dev/project", includePatterns, excludePatterns) assert.Equal(t, false, result) + + // also checking for uninitialized case which is going to be the scenario when not specified in yaml file + var ip []string + var ep []string + result = MatchIncludeExcludePatternsToFile("/projects/dev/test1", ip, ep) + assert.Equal(t, false, result) + } func TestGetPatternsRelativeToRepo(t *testing.T) { diff --git a/libs/digger_config/yaml.go b/libs/digger_config/yaml.go index d7c18e38..21d3f8d1 100644 --- a/libs/digger_config/yaml.go +++ b/libs/digger_config/yaml.go @@ -26,22 +26,22 @@ type DependencyConfigurationYaml struct { } type ProjectYaml struct { - Name string `yaml:"name"` - Dir string `yaml:"dir"` - Workspace string `yaml:"workspace"` - Terragrunt bool `yaml:"terragrunt"` - OpenTofu bool `yaml:"opentofu"` - Pulumi bool `yaml:"pulumi"` - Workflow string `yaml:"workflow"` - WorkflowFile string `yaml:"workflow_file"` - IncludePatterns []string `yaml:"include_patterns,omitempty"` - ExcludePatterns []string `yaml:"exclude_patterns,omitempty"` - DependencyProjects []string `yaml:"depends_on,omitempty"` - DriftDetection *bool `yaml:"drift_detection,omitempty"` - AwsRoleToAssume *AssumeRoleForProjectConfig `yaml:"aws_role_to_assume,omitempty"` - Generated bool `yaml:"generated"` - AwsCognitoOidcConfig *AwsCognitoOidcConfig `yaml:"aws_cognito_oidc,omitempty"` - PulumiStack string `yaml:"pulumi_stack"` + Name string `yaml:"name"` + Dir string `yaml:"dir"` + Workspace string `yaml:"workspace"` + Terragrunt bool `yaml:"terragrunt"` + OpenTofu bool `yaml:"opentofu"` + Pulumi bool `yaml:"pulumi"` + Workflow string `yaml:"workflow"` + WorkflowFile string `yaml:"workflow_file"` + IncludePatterns []string `yaml:"include_patterns,omitempty"` + ExcludePatterns []string `yaml:"exclude_patterns,omitempty"` + DependencyProjects []string `yaml:"depends_on,omitempty"` + DriftDetection *bool `yaml:"drift_detection,omitempty"` + AwsRoleToAssume *AssumeRoleForProjectConfig `yaml:"aws_role_to_assume,omitempty"` + Generated bool `yaml:"generated"` + AwsCognitoOidcConfig *AwsCognitoOidcConfig `yaml:"aws_cognito_oidc,omitempty"` + PulumiStack string `yaml:"pulumi_stack"` } type WorkflowYaml struct { @@ -92,8 +92,10 @@ type EnvVarYaml struct { type BlockYaml struct { // these flags are only for terraform and opentofu - Include string `yaml:"include"` - Exclude string `yaml:"exclude"` + Include string `yaml:"include"` + Exclude string `yaml:"exclude"` + IncludePatterns []string `yaml:"include_patterns,omitempty"` + ExcludePatterns []string `yaml:"exclude_patterns,omitempty"` // these flags are only for terragrunt Terragrunt bool `yaml:"terragrunt"` @@ -103,11 +105,12 @@ type BlockYaml struct { OpenTofu bool `yaml:"opentofu"` // common flags - BlockName string `yaml:"block_name"` - Workflow string `yaml:"workflow"` - WorkflowFile string `yaml:"workflow_file"` - AwsRoleToAssume *AssumeRoleForProjectConfig `yaml:"aws_role_to_assume,omitempty"` - AwsCognitoOidcConfig *AwsCognitoOidcConfig `yaml:"aws_cognito_oidc,omitempty"` + Workspace string `yaml:"workspace"` + BlockName string `yaml:"block_name"` + Workflow string `yaml:"workflow"` + WorkflowFile string `yaml:"workflow_file"` + AwsRoleToAssume *AssumeRoleForProjectConfig `yaml:"aws_role_to_assume,omitempty"` + AwsCognitoOidcConfig *AwsCognitoOidcConfig `yaml:"aws_cognito_oidc,omitempty"` } type AssumeRoleForProjectConfig struct { @@ -117,9 +120,9 @@ type AssumeRoleForProjectConfig struct { } type AwsCognitoOidcConfig struct { - AwsAccountId string `yaml:"aws_account_id"` - AwsRegion string `yaml:"aws_region,omitempty"` - CognitoPoolId string `yaml:"cognito_identity_pool_id"` + AwsAccountId string `yaml:"aws_account_id"` + AwsRegion string `yaml:"aws_region,omitempty"` + CognitoPoolId string `yaml:"cognito_identity_pool_id"` SessionDuration int `yaml:"session_duration"` } @@ -130,7 +133,7 @@ type GenerateProjectsConfigYaml struct { Blocks []BlockYaml `yaml:"blocks"` TerragruntParsingConfig *TerragruntParsingConfig `yaml:"terragrunt_parsing,omitempty"` AwsRoleToAssume *AssumeRoleForProjectConfig `yaml:"aws_role_to_assume,omitempty"` - AwsCognitoOidcConfig *AwsCognitoOidcConfig `yaml:"aws_cognito_oidc,omitempty"` + AwsCognitoOidcConfig *AwsCognitoOidcConfig `yaml:"aws_cognito_oidc,omitempty"` } type TerragruntParsingConfig struct { @@ -159,7 +162,7 @@ type TerragruntParsingConfig struct { ExecutionOrderGroups *bool `yaml:"executionOrderGroups"` WorkflowFile string `yaml:"workflow_file"` AwsRoleToAssume *AssumeRoleForProjectConfig `yaml:"aws_role_to_assume,omitempty"` - AwsCognitoOidcConfig *AwsCognitoOidcConfig `yaml:"aws_cognito_oidc,omitempty"` + AwsCognitoOidcConfig *AwsCognitoOidcConfig `yaml:"aws_cognito_oidc,omitempty"` } func (p *ProjectYaml) UnmarshalYAML(unmarshal func(interface{}) error) error {