From ee60848932c52829abfcc9d1392964b5c8a18d33 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Wed, 29 Jan 2025 23:20:52 +0500 Subject: [PATCH 01/20] symlinks --- internal/exec/go_getter_utils.go | 46 ++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index badfb3a67..089f2fe6d 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -99,6 +99,15 @@ func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) { return "", false, fmt.Errorf("invalid GitHub URL %q", parsedURL.Path) } + if !strings.Contains(parsedURL.Path, "//") { + // If it ends with .git treat it as wanting the entire repo + if strings.HasSuffix(parsedURL.Path, ".git") || + len(parts) == 3 { // means /owner/repo only + u.LogDebug(d.AtmosConfig, "Detected top-level repo with no subdir: appending '//.\n") + parsedURL.Path = parsedURL.Path + "//." + } + } + atmosGitHubToken := os.Getenv("ATMOS_GITHUB_TOKEN") gitHubToken := os.Getenv("GITHUB_TOKEN") @@ -158,7 +167,6 @@ func GoGetterGet( ) error { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - // Register custom detectors RegisterCustomDetectors(atmosConfig) @@ -168,8 +176,12 @@ func GoGetterGet( // Destination where the files will be stored. This will create the directory if it doesn't exist Dst: dest, Mode: clientMode, + Getters: map[string]getter.Getter{ + // Overriding 'git' + "git": &CustomGitGetter{}, + }, } - + fmt.Println(src, "src") if err := client.Get(); err != nil { return err } @@ -177,6 +189,36 @@ func GoGetterGet( return nil } +// CustomGitGetter is a custom getter for git (git::) that removes symlinks +type CustomGitGetter struct { + getter.GitGetter +} + +// Implements the custom getter logic removing symlinks +func (c *CustomGitGetter) Get(dst string, url *url.URL) error { + // Normal clone + if err := c.GitGetter.Get(dst, url); err != nil { + return err + } + // Remove symlinks + return removeSymlinks(dst) +} + +// removeSymlinks walks the destination directory and removes any symlinks +// it encounters. +func removeSymlinks(root string) error { + return filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.Mode()&os.ModeSymlink != 0 { + // It's a symlink, remove it + return os.Remove(path) + } + return nil + }) +} + // DownloadDetectFormatAndParseFile downloads a remote file, detects the format of the file (JSON, YAML, HCL) and parses the file into a Go type func DownloadDetectFormatAndParseFile(atmosConfig schema.AtmosConfiguration, file string) (any, error) { tempDir := os.TempDir() From 940d8769405a155eed76755521b98547a5d759e2 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Wed, 29 Jan 2025 23:32:21 +0500 Subject: [PATCH 02/20] comment --- internal/exec/go_getter_utils.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index 089f2fe6d..76c364290 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -181,7 +181,6 @@ func GoGetterGet( "git": &CustomGitGetter{}, }, } - fmt.Println(src, "src") if err := client.Get(); err != nil { return err } From 16a72f34e7032b44978d301fb900fc3470a98569 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Wed, 29 Jan 2025 23:35:15 +0500 Subject: [PATCH 03/20] log warning --- internal/exec/go_getter_utils.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index 76c364290..001deaab5 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -211,6 +211,7 @@ func removeSymlinks(root string) error { return err } if info.Mode()&os.ModeSymlink != 0 { + u.LogWarning(schema.AtmosConfiguration{}, fmt.Sprintf("Removing symlink: %s", path)) // It's a symlink, remove it return os.Remove(path) } From f9b3348d6550f3cf4641964da68aeb657e7ada9c Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Thu, 30 Jan 2025 00:15:49 +0500 Subject: [PATCH 04/20] adding back the rest getters --- internal/exec/go_getter_utils.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index 001deaab5..1db480e15 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -157,6 +157,15 @@ func RegisterCustomDetectors(atmosConfig schema.AtmosConfiguration) { ) } +func RegisterCustomGetters(atmosConfig schema.AtmosConfiguration) { + getter.Detectors = append( + []getter.Detector{ + &CustomGitHubDetector{AtmosConfig: atmosConfig}, + }, + getter.Detectors..., + ) +} + // GoGetterGet downloads packages (files and folders) from different sources using `go-getter` and saves them into the destination func GoGetterGet( atmosConfig schema.AtmosConfiguration, @@ -178,7 +187,14 @@ func GoGetterGet( Mode: clientMode, Getters: map[string]getter.Getter{ // Overriding 'git' - "git": &CustomGitGetter{}, + "git": &CustomGitGetter{}, + "file": &getter.FileGetter{}, + "hg": &getter.HgGetter{}, + "http": &getter.HttpGetter{}, + "https": &getter.HttpGetter{}, + // "s3": &getter.S3Getter{}, // add as needed + // "gcs": &getter.GCSGetter{}, + }, } if err := client.Get(); err != nil { From ed1af3d160740534970120835f2a7ebf8e4fef4a Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Fri, 31 Jan 2025 00:20:47 +0500 Subject: [PATCH 05/20] fix for windows path --- internal/exec/go_getter_utils.go | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index 1db480e15..aee7c8c7a 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -70,6 +70,7 @@ func IsValidScheme(scheme string) bool { // do a git-based clone with a token. type CustomGitHubDetector struct { AtmosConfig schema.AtmosConfiguration + source string } // Detect implements the getter.Detector interface for go-getter v1. @@ -99,11 +100,10 @@ func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) { return "", false, fmt.Errorf("invalid GitHub URL %q", parsedURL.Path) } - if !strings.Contains(parsedURL.Path, "//") { - // If it ends with .git treat it as wanting the entire repo - if strings.HasSuffix(parsedURL.Path, ".git") || - len(parts) == 3 { // means /owner/repo only - u.LogDebug(d.AtmosConfig, "Detected top-level repo with no subdir: appending '//.\n") + if !strings.Contains(d.source, "//") { + // means user typed something like "github.com/org/repo.git" with NO subdir + if strings.HasSuffix(parsedURL.Path, ".git") || len(parts) == 3 { + u.LogDebug(d.AtmosConfig, "Detected top-level repo with no subdir: appending '//.'\n") parsedURL.Path = parsedURL.Path + "//." } } @@ -148,19 +148,10 @@ func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) { // RegisterCustomDetectors prepends the custom detector so it runs before // the built-in ones. Any code that calls go-getter should invoke this. -func RegisterCustomDetectors(atmosConfig schema.AtmosConfiguration) { +func RegisterCustomDetectors(atmosConfig schema.AtmosConfiguration, source string) { getter.Detectors = append( []getter.Detector{ - &CustomGitHubDetector{AtmosConfig: atmosConfig}, - }, - getter.Detectors..., - ) -} - -func RegisterCustomGetters(atmosConfig schema.AtmosConfiguration) { - getter.Detectors = append( - []getter.Detector{ - &CustomGitHubDetector{AtmosConfig: atmosConfig}, + &CustomGitHubDetector{AtmosConfig: atmosConfig, source: source}, }, getter.Detectors..., ) @@ -176,8 +167,12 @@ func GoGetterGet( ) error { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - // Register custom detectors - RegisterCustomDetectors(atmosConfig) + + // Register custom detectors, passing the original `src` to the CustomGitHubDetector. + // go-getter typically strips subdirectories before calling the detector, so the + // unaltered source is needed to identify whether a top-level repository or a + // subdirectory was specified (e.g., for appending "//." only when no subdir is present). + RegisterCustomDetectors(atmosConfig, src) client := &getter.Client{ Ctx: ctx, From 5a6789e65f7d5b5ebfe3ed5bb13586aa19b74012 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Sun, 2 Feb 2025 00:42:34 +0500 Subject: [PATCH 06/20] globs --- internal/exec/copy_glob.go | 255 ++++++++++++++++++++ internal/exec/vendor_model.go | 2 +- tests/fixtures/scenarios/vendor/vendor.yaml | 11 + tests/test-cases/demo-stacks.yaml | 9 + 4 files changed, 276 insertions(+), 1 deletion(-) create mode 100644 internal/exec/copy_glob.go diff --git a/internal/exec/copy_glob.go b/internal/exec/copy_glob.go new file mode 100644 index 000000000..858d8bd69 --- /dev/null +++ b/internal/exec/copy_glob.go @@ -0,0 +1,255 @@ +package exec + +import ( + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/cloudposse/atmos/pkg/schema" + u "github.com/cloudposse/atmos/pkg/utils" + cp "github.com/otiai10/copy" // Using the optimized copy library when no filtering is required. +) + +// copyFile copies a single file from src to dst while preserving file permissions. +func copyFile(atmosConfig schema.AtmosConfiguration, src, dst string) error { + sourceFile, err := os.Open(src) + if err != nil { + return fmt.Errorf("opening source file %q: %w", src, err) + } + defer sourceFile.Close() + + if err := os.MkdirAll(filepath.Dir(dst), os.ModePerm); err != nil { + return fmt.Errorf("creating destination directory for %q: %w", dst, err) + } + + destinationFile, err := os.Create(dst) + if err != nil { + return fmt.Errorf("creating destination file %q: %w", dst, err) + } + defer destinationFile.Close() + + if _, err := io.Copy(destinationFile, sourceFile); err != nil { + return fmt.Errorf("copying content from %q to %q: %w", src, dst, err) + } + + info, err := os.Stat(src) + if err != nil { + return fmt.Errorf("getting file info for %q: %w", src, err) + } + if err := os.Chmod(dst, info.Mode()); err != nil { + return fmt.Errorf("setting permissions on %q: %w", dst, err) + } + return nil +} + +// skipFunc determines whether to skip a file/directory based on its relative path to baseDir. +// If an error occurs during matching for an exclusion or inclusion pattern, it logs the error and proceeds. +func skipFunc(atmosConfig schema.AtmosConfiguration, info os.FileInfo, srcPath, baseDir string, excluded, included []string) (bool, error) { + if info.Name() == ".git" { + return true, nil + } + relPath, err := filepath.Rel(baseDir, srcPath) + if err != nil { + u.LogTrace(atmosConfig, fmt.Sprintf("Error computing relative path for %q: %v", srcPath, err)) + return true, nil // treat error as a signal to skip + } + relPath = filepath.ToSlash(relPath) + + // Process exclusion patterns. + for _, pattern := range excluded { + matched, err := u.PathMatch(pattern, relPath) + if err != nil { + u.LogTrace(atmosConfig, fmt.Sprintf("Error matching exclusion pattern %q with %q: %v", pattern, relPath, err)) + continue + } else if matched { + u.LogTrace(atmosConfig, fmt.Sprintf("Excluding %q because it matches exclusion pattern %q", relPath, pattern)) + return true, nil + } + } + + // Process inclusion patterns (only for non-directory files). + if len(included) > 0 && !info.IsDir() { + matchedAny := false + for _, pattern := range included { + matched, err := u.PathMatch(pattern, relPath) + if err != nil { + u.LogTrace(atmosConfig, fmt.Sprintf("Error matching inclusion pattern %q with %q: %v", pattern, relPath, err)) + continue + } else if matched { + u.LogTrace(atmosConfig, fmt.Sprintf("Including %q because it matches inclusion pattern %q", relPath, pattern)) + matchedAny = true + break + } + } + if !matchedAny { + u.LogTrace(atmosConfig, fmt.Sprintf("Excluding %q because it does not match any inclusion pattern", relPath)) + return true, nil + } + } + return false, nil +} + +// copyDirRecursive recursively copies srcDir to dstDir using skipFunc filtering. +func copyDirRecursive(atmosConfig schema.AtmosConfiguration, srcDir, dstDir, baseDir string, excluded, included []string) error { + entries, err := os.ReadDir(srcDir) + if err != nil { + return fmt.Errorf("reading directory %q: %w", srcDir, err) + } + for _, entry := range entries { + srcPath := filepath.Join(srcDir, entry.Name()) + dstPath := filepath.Join(dstDir, entry.Name()) + + info, err := entry.Info() + if err != nil { + return fmt.Errorf("getting info for %q: %w", srcPath, err) + } + + skip, err := skipFunc(atmosConfig, info, srcPath, baseDir, excluded, included) + if err != nil { + return err + } + if skip { + continue + } + + // Skip symlinks. + if info.Mode()&os.ModeSymlink != 0 { + u.LogTrace(atmosConfig, fmt.Sprintf("Skipping symlink: %q", srcPath)) + continue + } + + if info.IsDir() { + if err := os.MkdirAll(dstPath, info.Mode()); err != nil { + return fmt.Errorf("creating directory %q: %w", dstPath, err) + } + if err := copyDirRecursive(atmosConfig, srcPath, dstPath, baseDir, excluded, included); err != nil { + return err + } + } else { + if err := copyFile(atmosConfig, srcPath, dstPath); err != nil { + return err + } + } + } + return nil +} + +// getMatchesForPattern returns files/directories matching a pattern relative to sourceDir. +// If no matches are found, it logs a trace and returns an empty slice. +// When the pattern ends with "/*", it retries with a recursive "/**" variant. +func getMatchesForPattern(atmosConfig schema.AtmosConfiguration, sourceDir, pattern string) ([]string, error) { + fullPattern := filepath.Join(sourceDir, pattern) + matches, err := u.GetGlobMatches(fullPattern) + if err != nil { + return nil, fmt.Errorf("error getting glob matches for %q: %w", fullPattern, err) + } + if len(matches) == 0 { + if strings.HasSuffix(pattern, "/*") { + recursivePattern := strings.TrimSuffix(pattern, "/*") + "/**" + fullRecursivePattern := filepath.Join(sourceDir, recursivePattern) + matches, err = u.GetGlobMatches(fullRecursivePattern) + if err != nil { + return nil, fmt.Errorf("error getting glob matches for recursive pattern %q: %w", fullRecursivePattern, err) + } + if len(matches) == 0 { + u.LogTrace(atmosConfig, fmt.Sprintf("No matches found for recursive pattern %q - target directory will be empty", fullRecursivePattern)) + return []string{}, nil + } + return matches, nil + } + u.LogTrace(atmosConfig, fmt.Sprintf("No matches found for pattern %q - target directory will be empty", fullPattern)) + return []string{}, nil + } + return matches, nil +} + +// copyToTargetWithPatterns copies the contents from sourceDir to targetPath, +// applying inclusion and exclusion patterns from the vendor source configuration. +// If sourceIsLocalFile is true and targetPath lacks an extension, the sanitized URI is appended. +// If no included paths are defined, all files (except those matching excluded paths) are copied. +// In the special case where neither inclusion nor exclusion patterns are defined, +// the optimized cp library (github.com/otiai10/copy) is used. +func copyToTargetWithPatterns( + atmosConfig schema.AtmosConfiguration, + sourceDir, targetPath string, + s *schema.AtmosVendorSource, + sourceIsLocalFile bool, + uri string, +) error { + if sourceIsLocalFile && filepath.Ext(targetPath) == "" { + targetPath = filepath.Join(targetPath, SanitizeFileName(uri)) + } + u.LogTrace(atmosConfig, fmt.Sprintf("Copying from %q to %q", sourceDir, targetPath)) + if err := os.MkdirAll(targetPath, os.ModePerm); err != nil { + return fmt.Errorf("creating target directory %q: %w", targetPath, err) + } + + // Optimization: if no inclusion and no exclusion patterns are defined, use the cp library for fast copying. + if len(s.IncludedPaths) == 0 && len(s.ExcludedPaths) == 0 { + u.LogTrace(atmosConfig, "No inclusion or exclusion patterns defined; using cp library for fast copy") + return cp.Copy(sourceDir, targetPath) + } + + // If inclusion patterns are provided, use them to determine which files to copy. + if len(s.IncludedPaths) > 0 { + filesToCopy := make(map[string]struct{}) + for _, pattern := range s.IncludedPaths { + matches, err := getMatchesForPattern(atmosConfig, sourceDir, pattern) + if err != nil { + u.LogTrace(atmosConfig, fmt.Sprintf("Warning: error getting matches for pattern %q: %v", pattern, err)) + continue + } + for _, match := range matches { + filesToCopy[match] = struct{}{} + } + } + if len(filesToCopy) == 0 { + u.LogTrace(atmosConfig, "No files matched the inclusion patterns - target directory will be empty") + return nil + } + for file := range filesToCopy { + relPath, err := filepath.Rel(sourceDir, file) + if err != nil { + return fmt.Errorf("computing relative path for %q: %w", file, err) + } + relPath = filepath.ToSlash(relPath) + skip := false + for _, ex := range s.ExcludedPaths { + matched, err := u.PathMatch(ex, relPath) + if err != nil { + u.LogTrace(atmosConfig, fmt.Sprintf("Error matching exclusion pattern %q with %q: %v", ex, relPath, err)) + continue + } else if matched { + u.LogTrace(atmosConfig, fmt.Sprintf("Excluding %q because it matches exclusion pattern %q", relPath, ex)) + skip = true + break + } + } + if skip { + continue + } + dstPath := filepath.Join(targetPath, relPath) + info, err := os.Stat(file) + if err != nil { + return fmt.Errorf("stating file %q: %w", file, err) + } + if info.IsDir() { + if err := copyDirRecursive(atmosConfig, file, dstPath, file, s.ExcludedPaths, nil); err != nil { + return err + } + } else { + if err := copyFile(atmosConfig, file, dstPath); err != nil { + return err + } + } + } + } else { + // No inclusion patterns defined; copy everything except those matching excluded items. + if err := copyDirRecursive(atmosConfig, sourceDir, targetPath, sourceDir, s.ExcludedPaths, s.IncludedPaths); err != nil { + return fmt.Errorf("error copying from %q to %q: %w", sourceDir, targetPath, err) + } + } + return nil +} diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go index 6ddf14e71..cbf257d67 100644 --- a/internal/exec/vendor_model.go +++ b/internal/exec/vendor_model.go @@ -307,7 +307,7 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.Atmos } } - if err := copyToTarget(atmosConfig, tempDir, p.targetPath, &p.atmosVendorSource, p.sourceIsLocalFile, p.uri); err != nil { + if err := copyToTargetWithPatterns(atmosConfig, tempDir, p.targetPath, &p.atmosVendorSource, p.sourceIsLocalFile, p.uri); err != nil { return installedPkgMsg{ err: fmt.Errorf("failed to copy package: %w", err), name: p.name, diff --git a/tests/fixtures/scenarios/vendor/vendor.yaml b/tests/fixtures/scenarios/vendor/vendor.yaml index 82cf62016..2c5ffce6c 100644 --- a/tests/fixtures/scenarios/vendor/vendor.yaml +++ b/tests/fixtures/scenarios/vendor/vendor.yaml @@ -51,3 +51,14 @@ spec: - "**/*.tftmpl" - "**/modules/**" excluded_paths: [] + + - component: "test globs" + source: "github.com/cloudposse/atmos.git" + included_paths: + - "**/{demo-library,demo-stacks}/**/*.{tf,md}" + excluded_paths: + - "**/demo-library/**/*.{tfvars,tf}" + targets: + - "components/library/" + tags: + - demo diff --git a/tests/test-cases/demo-stacks.yaml b/tests/test-cases/demo-stacks.yaml index 80923b7be..2d9dfffb1 100644 --- a/tests/test-cases/demo-stacks.yaml +++ b/tests/test-cases/demo-stacks.yaml @@ -193,4 +193,13 @@ tests: - "./components/terraform/vpc-src/outputs.tf" - "./components/terraform/vpc-src/variables.tf" - "./components/terraform/vpc-src/versions.tf" + - "./components/library/examples/demo-library/github/stargazers/README.md" + - "./components/library/examples/demo-library/ipinfo/README.md" + - "./components/library/examples/demo-library/weather/README.md" + - "./components/library/examples/demo-library/README.md" + - "./components/library/examples/demo-stacks/components/terraform/myapp/main.tf" + - "./components/library/examples/demo-stacks/components/terraform/myapp/outputs.tf" + - "./components/library/examples/demo-stacks/components/terraform/myapp/README.md" + - "./components/library/examples/demo-stacks/components/terraform/myapp/variables.tf" + - "./components/library/examples/demo-stacks/components/terraform/myapp/versions.tf" exit_code: 0 From e96dd7f3b180aa92d186ba8717ce9e78762d7a06 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Mon, 3 Feb 2025 20:36:00 +0500 Subject: [PATCH 07/20] removing atmosconfig from logging --- internal/exec/copy_glob.go | 30 +++++++++++++++--------------- internal/exec/go_getter_utils.go | 6 ++++-- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/internal/exec/copy_glob.go b/internal/exec/copy_glob.go index 858d8bd69..d04fdc46c 100644 --- a/internal/exec/copy_glob.go +++ b/internal/exec/copy_glob.go @@ -52,7 +52,7 @@ func skipFunc(atmosConfig schema.AtmosConfiguration, info os.FileInfo, srcPath, } relPath, err := filepath.Rel(baseDir, srcPath) if err != nil { - u.LogTrace(atmosConfig, fmt.Sprintf("Error computing relative path for %q: %v", srcPath, err)) + u.LogTrace(fmt.Sprintf("Error computing relative path for %q: %v", srcPath, err)) return true, nil // treat error as a signal to skip } relPath = filepath.ToSlash(relPath) @@ -61,10 +61,10 @@ func skipFunc(atmosConfig schema.AtmosConfiguration, info os.FileInfo, srcPath, for _, pattern := range excluded { matched, err := u.PathMatch(pattern, relPath) if err != nil { - u.LogTrace(atmosConfig, fmt.Sprintf("Error matching exclusion pattern %q with %q: %v", pattern, relPath, err)) + u.LogTrace(fmt.Sprintf("Error matching exclusion pattern %q with %q: %v", pattern, relPath, err)) continue } else if matched { - u.LogTrace(atmosConfig, fmt.Sprintf("Excluding %q because it matches exclusion pattern %q", relPath, pattern)) + u.LogTrace(fmt.Sprintf("Excluding %q because it matches exclusion pattern %q", relPath, pattern)) return true, nil } } @@ -75,16 +75,16 @@ func skipFunc(atmosConfig schema.AtmosConfiguration, info os.FileInfo, srcPath, for _, pattern := range included { matched, err := u.PathMatch(pattern, relPath) if err != nil { - u.LogTrace(atmosConfig, fmt.Sprintf("Error matching inclusion pattern %q with %q: %v", pattern, relPath, err)) + u.LogTrace(fmt.Sprintf("Error matching inclusion pattern %q with %q: %v", pattern, relPath, err)) continue } else if matched { - u.LogTrace(atmosConfig, fmt.Sprintf("Including %q because it matches inclusion pattern %q", relPath, pattern)) + u.LogTrace(fmt.Sprintf("Including %q because it matches inclusion pattern %q", relPath, pattern)) matchedAny = true break } } if !matchedAny { - u.LogTrace(atmosConfig, fmt.Sprintf("Excluding %q because it does not match any inclusion pattern", relPath)) + u.LogTrace(fmt.Sprintf("Excluding %q because it does not match any inclusion pattern", relPath)) return true, nil } } @@ -116,7 +116,7 @@ func copyDirRecursive(atmosConfig schema.AtmosConfiguration, srcDir, dstDir, bas // Skip symlinks. if info.Mode()&os.ModeSymlink != 0 { - u.LogTrace(atmosConfig, fmt.Sprintf("Skipping symlink: %q", srcPath)) + u.LogTrace(fmt.Sprintf("Skipping symlink: %q", srcPath)) continue } @@ -154,12 +154,12 @@ func getMatchesForPattern(atmosConfig schema.AtmosConfiguration, sourceDir, patt return nil, fmt.Errorf("error getting glob matches for recursive pattern %q: %w", fullRecursivePattern, err) } if len(matches) == 0 { - u.LogTrace(atmosConfig, fmt.Sprintf("No matches found for recursive pattern %q - target directory will be empty", fullRecursivePattern)) + u.LogTrace(fmt.Sprintf("No matches found for recursive pattern %q - target directory will be empty", fullRecursivePattern)) return []string{}, nil } return matches, nil } - u.LogTrace(atmosConfig, fmt.Sprintf("No matches found for pattern %q - target directory will be empty", fullPattern)) + u.LogTrace(fmt.Sprintf("No matches found for pattern %q - target directory will be empty", fullPattern)) return []string{}, nil } return matches, nil @@ -181,14 +181,14 @@ func copyToTargetWithPatterns( if sourceIsLocalFile && filepath.Ext(targetPath) == "" { targetPath = filepath.Join(targetPath, SanitizeFileName(uri)) } - u.LogTrace(atmosConfig, fmt.Sprintf("Copying from %q to %q", sourceDir, targetPath)) + u.LogTrace(fmt.Sprintf("Copying from %q to %q", sourceDir, targetPath)) if err := os.MkdirAll(targetPath, os.ModePerm); err != nil { return fmt.Errorf("creating target directory %q: %w", targetPath, err) } // Optimization: if no inclusion and no exclusion patterns are defined, use the cp library for fast copying. if len(s.IncludedPaths) == 0 && len(s.ExcludedPaths) == 0 { - u.LogTrace(atmosConfig, "No inclusion or exclusion patterns defined; using cp library for fast copy") + u.LogTrace("No inclusion or exclusion patterns defined; using cp library for fast copy") return cp.Copy(sourceDir, targetPath) } @@ -198,7 +198,7 @@ func copyToTargetWithPatterns( for _, pattern := range s.IncludedPaths { matches, err := getMatchesForPattern(atmosConfig, sourceDir, pattern) if err != nil { - u.LogTrace(atmosConfig, fmt.Sprintf("Warning: error getting matches for pattern %q: %v", pattern, err)) + u.LogTrace(fmt.Sprintf("Warning: error getting matches for pattern %q: %v", pattern, err)) continue } for _, match := range matches { @@ -206,7 +206,7 @@ func copyToTargetWithPatterns( } } if len(filesToCopy) == 0 { - u.LogTrace(atmosConfig, "No files matched the inclusion patterns - target directory will be empty") + u.LogTrace("No files matched the inclusion patterns - target directory will be empty") return nil } for file := range filesToCopy { @@ -219,10 +219,10 @@ func copyToTargetWithPatterns( for _, ex := range s.ExcludedPaths { matched, err := u.PathMatch(ex, relPath) if err != nil { - u.LogTrace(atmosConfig, fmt.Sprintf("Error matching exclusion pattern %q with %q: %v", ex, relPath, err)) + u.LogTrace(fmt.Sprintf("Error matching exclusion pattern %q with %q: %v", ex, relPath, err)) continue } else if matched { - u.LogTrace(atmosConfig, fmt.Sprintf("Excluding %q because it matches exclusion pattern %q", relPath, ex)) + u.LogTrace(fmt.Sprintf("Excluding %q because it matches exclusion pattern %q", relPath, ex)) skip = true break } diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index c3b055498..bbffce9cf 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -103,7 +103,7 @@ func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) { if !strings.Contains(d.source, "//") { // means user typed something like "github.com/org/repo.git" with NO subdir if strings.HasSuffix(parsedURL.Path, ".git") || len(parts) == 3 { - u.LogDebug(d.AtmosConfig, "Detected top-level repo with no subdir: appending '//.'\n") + u.LogDebug("Detected top-level repo with no subdir: appending '//.'\n") parsedURL.Path = parsedURL.Path + "//." } } @@ -222,7 +222,9 @@ func removeSymlinks(root string) error { return err } if info.Mode()&os.ModeSymlink != 0 { - u.LogWarning(schema.AtmosConfiguration{}, fmt.Sprintf("Removing symlink: %s", path)) + //Symlinks are removed for the entire repo, regardless if there are any subfolders specified + //Thus logging is disabled + //u.LogWarning(fmt.Sprintf("Removing symlink: %s", path)) // It's a symlink, remove it return os.Remove(path) } From 06f5f3bd762e4e221e44dfd018d3f65d93d2c708 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Mon, 3 Feb 2025 15:37:15 +0000 Subject: [PATCH 08/20] [autofix.ci] apply automated fixes --- internal/exec/go_getter_utils.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index bbffce9cf..ba1fc6279 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -222,9 +222,9 @@ func removeSymlinks(root string) error { return err } if info.Mode()&os.ModeSymlink != 0 { - //Symlinks are removed for the entire repo, regardless if there are any subfolders specified - //Thus logging is disabled - //u.LogWarning(fmt.Sprintf("Removing symlink: %s", path)) + // Symlinks are removed for the entire repo, regardless if there are any subfolders specified + // Thus logging is disabled + // u.LogWarning(fmt.Sprintf("Removing symlink: %s", path)) // It's a symlink, remove it return os.Remove(path) } From 82caa056392c9f8529a85bb88edbc247f338dd66 Mon Sep 17 00:00:00 2001 From: Listener430 <95291462+Listener430@users.noreply.github.com> Date: Thu, 6 Feb 2025 13:38:35 +0600 Subject: [PATCH 09/20] Update internal/exec/copy_glob.go Co-authored-by: Erik Osterman (CEO @ Cloud Posse) --- internal/exec/copy_glob.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/exec/copy_glob.go b/internal/exec/copy_glob.go index d04fdc46c..1a0bd2401 100644 --- a/internal/exec/copy_glob.go +++ b/internal/exec/copy_glob.go @@ -52,7 +52,7 @@ func skipFunc(atmosConfig schema.AtmosConfiguration, info os.FileInfo, srcPath, } relPath, err := filepath.Rel(baseDir, srcPath) if err != nil { - u.LogTrace(fmt.Sprintf("Error computing relative path for %q: %v", srcPath, err)) + l.LogDebug("Error computing relative path", 'srcPath', srcPath, 'err', err) return true, nil // treat error as a signal to skip } relPath = filepath.ToSlash(relPath) From b9d4e5a4680313919f83e3fe16682939dc5cea06 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Thu, 6 Feb 2025 13:01:49 +0500 Subject: [PATCH 10/20] logging to charmbracelet --- internal/exec/copy_glob.go | 47 +++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/internal/exec/copy_glob.go b/internal/exec/copy_glob.go index 1a0bd2401..e58048f55 100644 --- a/internal/exec/copy_glob.go +++ b/internal/exec/copy_glob.go @@ -7,13 +7,14 @@ import ( "path/filepath" "strings" + l "github.com/charmbracelet/log" "github.com/cloudposse/atmos/pkg/schema" u "github.com/cloudposse/atmos/pkg/utils" cp "github.com/otiai10/copy" // Using the optimized copy library when no filtering is required. ) // copyFile copies a single file from src to dst while preserving file permissions. -func copyFile(atmosConfig schema.AtmosConfiguration, src, dst string) error { +func copyFile(src, dst string) error { sourceFile, err := os.Open(src) if err != nil { return fmt.Errorf("opening source file %q: %w", src, err) @@ -46,13 +47,13 @@ func copyFile(atmosConfig schema.AtmosConfiguration, src, dst string) error { // skipFunc determines whether to skip a file/directory based on its relative path to baseDir. // If an error occurs during matching for an exclusion or inclusion pattern, it logs the error and proceeds. -func skipFunc(atmosConfig schema.AtmosConfiguration, info os.FileInfo, srcPath, baseDir string, excluded, included []string) (bool, error) { +func skipFunc(info os.FileInfo, srcPath, baseDir string, excluded, included []string) (bool, error) { if info.Name() == ".git" { return true, nil } relPath, err := filepath.Rel(baseDir, srcPath) if err != nil { - l.LogDebug("Error computing relative path", 'srcPath', srcPath, 'err', err) + l.Debug("Error computing relative path", "srcPath", srcPath, "error", err) return true, nil // treat error as a signal to skip } relPath = filepath.ToSlash(relPath) @@ -61,10 +62,10 @@ func skipFunc(atmosConfig schema.AtmosConfiguration, info os.FileInfo, srcPath, for _, pattern := range excluded { matched, err := u.PathMatch(pattern, relPath) if err != nil { - u.LogTrace(fmt.Sprintf("Error matching exclusion pattern %q with %q: %v", pattern, relPath, err)) + l.Debug("Error matching exclusion pattern", "pattern", pattern, "path", relPath, "error", err) continue } else if matched { - u.LogTrace(fmt.Sprintf("Excluding %q because it matches exclusion pattern %q", relPath, pattern)) + l.Debug("Excluding path due to exclusion pattern", "path", relPath, "pattern", pattern) return true, nil } } @@ -75,16 +76,16 @@ func skipFunc(atmosConfig schema.AtmosConfiguration, info os.FileInfo, srcPath, for _, pattern := range included { matched, err := u.PathMatch(pattern, relPath) if err != nil { - u.LogTrace(fmt.Sprintf("Error matching inclusion pattern %q with %q: %v", pattern, relPath, err)) + l.Debug("Error matching inclusion pattern", "pattern", pattern, "path", relPath, "error", err) continue } else if matched { - u.LogTrace(fmt.Sprintf("Including %q because it matches inclusion pattern %q", relPath, pattern)) + l.Debug("Including path due to inclusion pattern", "path", relPath, "pattern", pattern) matchedAny = true break } } if !matchedAny { - u.LogTrace(fmt.Sprintf("Excluding %q because it does not match any inclusion pattern", relPath)) + l.Debug("Excluding path because it does not match any inclusion pattern", "path", relPath) return true, nil } } @@ -106,7 +107,7 @@ func copyDirRecursive(atmosConfig schema.AtmosConfiguration, srcDir, dstDir, bas return fmt.Errorf("getting info for %q: %w", srcPath, err) } - skip, err := skipFunc(atmosConfig, info, srcPath, baseDir, excluded, included) + skip, err := skipFunc(info, srcPath, baseDir, excluded, included) if err != nil { return err } @@ -116,7 +117,7 @@ func copyDirRecursive(atmosConfig schema.AtmosConfiguration, srcDir, dstDir, bas // Skip symlinks. if info.Mode()&os.ModeSymlink != 0 { - u.LogTrace(fmt.Sprintf("Skipping symlink: %q", srcPath)) + l.Debug("Skipping symlink", "path", srcPath) continue } @@ -128,7 +129,7 @@ func copyDirRecursive(atmosConfig schema.AtmosConfiguration, srcDir, dstDir, bas return err } } else { - if err := copyFile(atmosConfig, srcPath, dstPath); err != nil { + if err := copyFile(srcPath, dstPath); err != nil { return err } } @@ -137,9 +138,9 @@ func copyDirRecursive(atmosConfig schema.AtmosConfiguration, srcDir, dstDir, bas } // getMatchesForPattern returns files/directories matching a pattern relative to sourceDir. -// If no matches are found, it logs a trace and returns an empty slice. +// If no matches are found, it logs a debug message and returns an empty slice. // When the pattern ends with "/*", it retries with a recursive "/**" variant. -func getMatchesForPattern(atmosConfig schema.AtmosConfiguration, sourceDir, pattern string) ([]string, error) { +func getMatchesForPattern(sourceDir, pattern string) ([]string, error) { fullPattern := filepath.Join(sourceDir, pattern) matches, err := u.GetGlobMatches(fullPattern) if err != nil { @@ -154,12 +155,12 @@ func getMatchesForPattern(atmosConfig schema.AtmosConfiguration, sourceDir, patt return nil, fmt.Errorf("error getting glob matches for recursive pattern %q: %w", fullRecursivePattern, err) } if len(matches) == 0 { - u.LogTrace(fmt.Sprintf("No matches found for recursive pattern %q - target directory will be empty", fullRecursivePattern)) + l.Debug("No matches found for recursive pattern; target directory will be empty", "pattern", fullRecursivePattern) return []string{}, nil } return matches, nil } - u.LogTrace(fmt.Sprintf("No matches found for pattern %q - target directory will be empty", fullPattern)) + l.Debug("No matches found for pattern; target directory will be empty", "pattern", fullPattern) return []string{}, nil } return matches, nil @@ -181,14 +182,14 @@ func copyToTargetWithPatterns( if sourceIsLocalFile && filepath.Ext(targetPath) == "" { targetPath = filepath.Join(targetPath, SanitizeFileName(uri)) } - u.LogTrace(fmt.Sprintf("Copying from %q to %q", sourceDir, targetPath)) + l.Debug("Copying files", "source", sourceDir, "target", targetPath) if err := os.MkdirAll(targetPath, os.ModePerm); err != nil { return fmt.Errorf("creating target directory %q: %w", targetPath, err) } // Optimization: if no inclusion and no exclusion patterns are defined, use the cp library for fast copying. if len(s.IncludedPaths) == 0 && len(s.ExcludedPaths) == 0 { - u.LogTrace("No inclusion or exclusion patterns defined; using cp library for fast copy") + l.Debug("No inclusion or exclusion patterns defined; using cp library for fast copy") return cp.Copy(sourceDir, targetPath) } @@ -196,9 +197,9 @@ func copyToTargetWithPatterns( if len(s.IncludedPaths) > 0 { filesToCopy := make(map[string]struct{}) for _, pattern := range s.IncludedPaths { - matches, err := getMatchesForPattern(atmosConfig, sourceDir, pattern) + matches, err := getMatchesForPattern(sourceDir, pattern) if err != nil { - u.LogTrace(fmt.Sprintf("Warning: error getting matches for pattern %q: %v", pattern, err)) + l.Debug("Warning: error getting matches for pattern", "pattern", pattern, "error", err) continue } for _, match := range matches { @@ -206,7 +207,7 @@ func copyToTargetWithPatterns( } } if len(filesToCopy) == 0 { - u.LogTrace("No files matched the inclusion patterns - target directory will be empty") + l.Debug("No files matched the inclusion patterns - target directory will be empty") return nil } for file := range filesToCopy { @@ -219,10 +220,10 @@ func copyToTargetWithPatterns( for _, ex := range s.ExcludedPaths { matched, err := u.PathMatch(ex, relPath) if err != nil { - u.LogTrace(fmt.Sprintf("Error matching exclusion pattern %q with %q: %v", ex, relPath, err)) + l.Debug("Error matching exclusion pattern", "pattern", ex, "path", relPath, "error", err) continue } else if matched { - u.LogTrace(fmt.Sprintf("Excluding %q because it matches exclusion pattern %q", relPath, ex)) + l.Debug("Excluding file due to exclusion pattern", "file", relPath, "pattern", ex) skip = true break } @@ -240,7 +241,7 @@ func copyToTargetWithPatterns( return err } } else { - if err := copyFile(atmosConfig, file, dstPath); err != nil { + if err := copyFile(file, dstPath); err != nil { return err } } From 0b8e5fa46fbe1033ecb8f8a207d53a504924af1d Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Thu, 6 Feb 2025 14:03:27 +0500 Subject: [PATCH 11/20] renaming skipFunc --- internal/exec/copy_glob.go | 17 ++++++++--------- internal/exec/vendor_model.go | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/internal/exec/copy_glob.go b/internal/exec/copy_glob.go index e58048f55..e9460009f 100644 --- a/internal/exec/copy_glob.go +++ b/internal/exec/copy_glob.go @@ -45,9 +45,9 @@ func copyFile(src, dst string) error { return nil } -// skipFunc determines whether to skip a file/directory based on its relative path to baseDir. +// shouldSkipEntry determines whether to skip a file/directory based on its relative path to baseDir. // If an error occurs during matching for an exclusion or inclusion pattern, it logs the error and proceeds. -func skipFunc(info os.FileInfo, srcPath, baseDir string, excluded, included []string) (bool, error) { +func shouldSkipEntry(info os.FileInfo, srcPath, baseDir string, excluded, included []string) (bool, error) { if info.Name() == ".git" { return true, nil } @@ -92,8 +92,8 @@ func skipFunc(info os.FileInfo, srcPath, baseDir string, excluded, included []st return false, nil } -// copyDirRecursive recursively copies srcDir to dstDir using skipFunc filtering. -func copyDirRecursive(atmosConfig schema.AtmosConfiguration, srcDir, dstDir, baseDir string, excluded, included []string) error { +// copyDirRecursive recursively copies srcDir to dstDir using shouldSkipEntry filtering. +func copyDirRecursive(srcDir, dstDir, baseDir string, excluded, included []string) error { entries, err := os.ReadDir(srcDir) if err != nil { return fmt.Errorf("reading directory %q: %w", srcDir, err) @@ -107,7 +107,7 @@ func copyDirRecursive(atmosConfig schema.AtmosConfiguration, srcDir, dstDir, bas return fmt.Errorf("getting info for %q: %w", srcPath, err) } - skip, err := skipFunc(info, srcPath, baseDir, excluded, included) + skip, err := shouldSkipEntry(info, srcPath, baseDir, excluded, included) if err != nil { return err } @@ -125,7 +125,7 @@ func copyDirRecursive(atmosConfig schema.AtmosConfiguration, srcDir, dstDir, bas if err := os.MkdirAll(dstPath, info.Mode()); err != nil { return fmt.Errorf("creating directory %q: %w", dstPath, err) } - if err := copyDirRecursive(atmosConfig, srcPath, dstPath, baseDir, excluded, included); err != nil { + if err := copyDirRecursive(srcPath, dstPath, baseDir, excluded, included); err != nil { return err } } else { @@ -173,7 +173,6 @@ func getMatchesForPattern(sourceDir, pattern string) ([]string, error) { // In the special case where neither inclusion nor exclusion patterns are defined, // the optimized cp library (github.com/otiai10/copy) is used. func copyToTargetWithPatterns( - atmosConfig schema.AtmosConfiguration, sourceDir, targetPath string, s *schema.AtmosVendorSource, sourceIsLocalFile bool, @@ -237,7 +236,7 @@ func copyToTargetWithPatterns( return fmt.Errorf("stating file %q: %w", file, err) } if info.IsDir() { - if err := copyDirRecursive(atmosConfig, file, dstPath, file, s.ExcludedPaths, nil); err != nil { + if err := copyDirRecursive(file, dstPath, file, s.ExcludedPaths, nil); err != nil { return err } } else { @@ -248,7 +247,7 @@ func copyToTargetWithPatterns( } } else { // No inclusion patterns defined; copy everything except those matching excluded items. - if err := copyDirRecursive(atmosConfig, sourceDir, targetPath, sourceDir, s.ExcludedPaths, s.IncludedPaths); err != nil { + if err := copyDirRecursive(sourceDir, targetPath, sourceDir, s.ExcludedPaths, s.IncludedPaths); err != nil { return fmt.Errorf("error copying from %q to %q: %w", sourceDir, targetPath, err) } } diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go index b48331144..c12b0045b 100644 --- a/internal/exec/vendor_model.go +++ b/internal/exec/vendor_model.go @@ -307,7 +307,7 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.Atmos } } - if err := copyToTargetWithPatterns(atmosConfig, tempDir, p.targetPath, &p.atmosVendorSource, p.sourceIsLocalFile, p.uri); err != nil { + if err := copyToTargetWithPatterns(tempDir, p.targetPath, &p.atmosVendorSource, p.sourceIsLocalFile, p.uri); err != nil { return installedPkgMsg{ err: fmt.Errorf("failed to copy package: %w", err), name: p.name, From e8133be3af6a336c43a7c1687d6505e69cc7baab Mon Sep 17 00:00:00 2001 From: Listener430 <95291462+Listener430@users.noreply.github.com> Date: Thu, 6 Feb 2025 15:37:56 +0600 Subject: [PATCH 12/20] Update internal/exec/go_getter_utils.go Co-authored-by: Erik Osterman (CEO @ Cloud Posse) --- internal/exec/go_getter_utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index ba1fc6279..c8f9a84b9 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -214,7 +214,7 @@ func (c *CustomGitGetter) Get(dst string, url *url.URL) error { return removeSymlinks(dst) } -// removeSymlinks walks the destination directory and removes any symlinks +// removeSymlinks walks the directory and removes any symlinks // it encounters. func removeSymlinks(root string) error { return filepath.Walk(root, func(path string, info os.FileInfo, err error) error { From 84794096f2e93e673301859c43e81f558dc662bf Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Thu, 6 Feb 2025 16:57:00 +0500 Subject: [PATCH 13/20] new testcase --- tests/fixtures/scenarios/vendor/vendor.yaml | 10 ++++++++++ tests/test-cases/demo-stacks.yaml | 1 + 2 files changed, 11 insertions(+) diff --git a/tests/fixtures/scenarios/vendor/vendor.yaml b/tests/fixtures/scenarios/vendor/vendor.yaml index 2c5ffce6c..ba3c016f5 100644 --- a/tests/fixtures/scenarios/vendor/vendor.yaml +++ b/tests/fixtures/scenarios/vendor/vendor.yaml @@ -62,3 +62,13 @@ spec: - "components/library/" tags: - demo + + - component: "test globs without double stars upfront" + source: "github.com/cloudposse/atmos.git//examples/demo-library?ref={{.Version}}" + included_paths: + - "/weather/*.md" + version: "main" + targets: + - "components/library/" + tags: + - demo diff --git a/tests/test-cases/demo-stacks.yaml b/tests/test-cases/demo-stacks.yaml index 4a0ca1a4c..367742db9 100644 --- a/tests/test-cases/demo-stacks.yaml +++ b/tests/test-cases/demo-stacks.yaml @@ -192,4 +192,5 @@ tests: - "./components/library/examples/demo-stacks/components/terraform/myapp/README.md" - "./components/library/examples/demo-stacks/components/terraform/myapp/variables.tf" - "./components/library/examples/demo-stacks/components/terraform/myapp/versions.tf" + - "./components/library/weather/README.md" exit_code: 0 From c4adfb20f5beaaf096c132f6f5a2c19ddd7845db Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Thu, 6 Feb 2025 18:35:48 +0500 Subject: [PATCH 14/20] added depth=1 --- internal/exec/go_getter_utils.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index 142c32cd5..4a31b3dab 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -136,6 +136,13 @@ func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) { } } + // Ensure that the query parameter "depth" is set to "1" if not already present. + q := parsedURL.Query() + if q.Get("depth") == "" { + q.Set("depth", "1") + } + parsedURL.RawQuery = q.Encode() + finalURL := "git::" + parsedURL.String() return finalURL, true, nil From 7382cab94df827b9b2119ba1477d863e6b359a14 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Thu, 6 Feb 2025 19:09:26 +0500 Subject: [PATCH 15/20] depth --- internal/exec/go_getter_utils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index 4a31b3dab..0fc8f8a87 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -136,9 +136,9 @@ func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) { } } - // Ensure that the query parameter "depth" is set to "1" if not already present. + // Ensure that the "depth" parameter is set to "1" if the key does not exist. q := parsedURL.Query() - if q.Get("depth") == "" { + if _, exists := q["depth"]; !exists { q.Set("depth", "1") } parsedURL.RawQuery = q.Encode() From a9c8f48f262f10a953a6a5023d1f60892572b606 Mon Sep 17 00:00:00 2001 From: Listener430 <95291462+Listener430@users.noreply.github.com> Date: Thu, 6 Feb 2025 21:04:33 +0600 Subject: [PATCH 16/20] Update internal/exec/go_getter_utils.go Co-authored-by: Erik Osterman (CEO @ Cloud Posse) --- internal/exec/go_getter_utils.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index 0fc8f8a87..38797fb07 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -136,7 +136,11 @@ func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) { } } - // Ensure that the "depth" parameter is set to "1" if the key does not exist. + // Set "depth=1" for a shallow clone if not specified. + // In Go-Getter, "depth" controls how many revisions are cloned: + // - `depth=1` fetches only the latest commit (faster, less bandwidth). + // - `depth=` (empty) performs a full clone (default Git behavior). + // - `depth=N` clones the last N revisions. q := parsedURL.Query() if _, exists := q["depth"]; !exists { q.Set("depth", "1") From a54dafb5c5ce984971bc7d3781abc5a095e501fd Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Thu, 6 Feb 2025 15:05:27 +0000 Subject: [PATCH 17/20] [autofix.ci] apply automated fixes --- internal/exec/go_getter_utils.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index 38797fb07..f76699f58 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -136,11 +136,11 @@ func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) { } } - // Set "depth=1" for a shallow clone if not specified. - // In Go-Getter, "depth" controls how many revisions are cloned: - // - `depth=1` fetches only the latest commit (faster, less bandwidth). - // - `depth=` (empty) performs a full clone (default Git behavior). - // - `depth=N` clones the last N revisions. + // Set "depth=1" for a shallow clone if not specified. + // In Go-Getter, "depth" controls how many revisions are cloned: + // - `depth=1` fetches only the latest commit (faster, less bandwidth). + // - `depth=` (empty) performs a full clone (default Git behavior). + // - `depth=N` clones the last N revisions. q := parsedURL.Query() if _, exists := q["depth"]; !exists { q.Set("depth", "1") From 2bf586a5f3f1e3f2abb39e3e318afc4a9003572f Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Fri, 7 Feb 2025 22:20:59 +0500 Subject: [PATCH 18/20] excluded subfolders --- internal/exec/copy_glob.go | 153 +++++++++++++++++++++++++++++++++---- 1 file changed, 137 insertions(+), 16 deletions(-) diff --git a/internal/exec/copy_glob.go b/internal/exec/copy_glob.go index e9460009f..105fa50bd 100644 --- a/internal/exec/copy_glob.go +++ b/internal/exec/copy_glob.go @@ -56,21 +56,38 @@ func shouldSkipEntry(info os.FileInfo, srcPath, baseDir string, excluded, includ l.Debug("Error computing relative path", "srcPath", srcPath, "error", err) return true, nil // treat error as a signal to skip } + // Ensure uniform path separator. relPath = filepath.ToSlash(relPath) // Process exclusion patterns. + // For directories, check with and without a trailing slash. for _, pattern := range excluded { + // First check the plain relative path. matched, err := u.PathMatch(pattern, relPath) if err != nil { l.Debug("Error matching exclusion pattern", "pattern", pattern, "path", relPath, "error", err) continue - } else if matched { - l.Debug("Excluding path due to exclusion pattern", "path", relPath, "pattern", pattern) + } + if matched { + l.Debug("Excluding path due to exclusion pattern (plain match)", "path", relPath, "pattern", pattern) return true, nil } + // If it is a directory, also try matching with a trailing slash. + if info.IsDir() { + matched, err = u.PathMatch(pattern, relPath+"/") + if err != nil { + l.Debug("Error matching exclusion pattern with trailing slash", "pattern", pattern, "path", relPath+"/", "error", err) + continue + } + if matched { + l.Debug("Excluding directory due to exclusion pattern (with trailing slash)", "path", relPath+"/", "pattern", pattern) + return true, nil + } + } } // Process inclusion patterns (only for non-directory files). + // (Directories are generally picked up by the inclusion branch in copyToTargetWithPatterns.) if len(included) > 0 && !info.IsDir() { matchedAny := false for _, pattern := range included { @@ -78,7 +95,8 @@ func shouldSkipEntry(info os.FileInfo, srcPath, baseDir string, excluded, includ if err != nil { l.Debug("Error matching inclusion pattern", "pattern", pattern, "path", relPath, "error", err) continue - } else if matched { + } + if matched { l.Debug("Including path due to inclusion pattern", "path", relPath, "pattern", pattern) matchedAny = true break @@ -93,6 +111,7 @@ func shouldSkipEntry(info os.FileInfo, srcPath, baseDir string, excluded, includ } // copyDirRecursive recursively copies srcDir to dstDir using shouldSkipEntry filtering. +// This function is used in cases where the entire sourceDir is the base for relative paths. func copyDirRecursive(srcDir, dstDir, baseDir string, excluded, included []string) error { entries, err := os.ReadDir(srcDir) if err != nil { @@ -107,11 +126,13 @@ func copyDirRecursive(srcDir, dstDir, baseDir string, excluded, included []strin return fmt.Errorf("getting info for %q: %w", srcPath, err) } + // Check if this entry should be skipped. skip, err := shouldSkipEntry(info, srcPath, baseDir, excluded, included) if err != nil { return err } if skip { + l.Debug("Skipping entry", "srcPath", srcPath) continue } @@ -137,6 +158,80 @@ func copyDirRecursive(srcDir, dstDir, baseDir string, excluded, included []strin return nil } +// copyDirRecursiveWithPrefix recursively copies srcDir to dstDir while preserving the global relative path. +// Instead of using the local srcDir as the base for computing relative paths, this function uses the original +// source directory (globalBase) and an accumulated prefix that represents the relative path from globalBase. +func copyDirRecursiveWithPrefix(srcDir, dstDir, globalBase, prefix string, excluded []string) error { + entries, err := os.ReadDir(srcDir) + if err != nil { + return fmt.Errorf("reading directory %q: %w", srcDir, err) + } + for _, entry := range entries { + // Compute the full relative path from the original source. + fullRelPath := filepath.ToSlash(filepath.Join(prefix, entry.Name())) + srcPath := filepath.Join(srcDir, entry.Name()) + dstPath := filepath.Join(dstDir, entry.Name()) + + info, err := entry.Info() + if err != nil { + return fmt.Errorf("getting info for %q: %w", srcPath, err) + } + + // Skip .git directories. + if entry.Name() == ".git" { + l.Debug("Skipping .git directory", "path", fullRelPath) + continue + } + + // Check exclusion patterns using the full relative path. + skip := false + for _, pattern := range excluded { + // Check plain match. + matched, err := u.PathMatch(pattern, fullRelPath) + if err != nil { + l.Debug("Error matching exclusion pattern in prefix function", "pattern", pattern, "path", fullRelPath, "error", err) + continue + } + if matched { + l.Debug("Excluding (prefix) due to exclusion pattern (plain match)", "path", fullRelPath, "pattern", pattern) + skip = true + break + } + // For directories, also try with a trailing slash. + if info.IsDir() { + matched, err = u.PathMatch(pattern, fullRelPath+"/") + if err != nil { + l.Debug("Error matching exclusion pattern with trailing slash in prefix function", "pattern", pattern, "path", fullRelPath+"/", "error", err) + continue + } + if matched { + l.Debug("Excluding (prefix) due to exclusion pattern (with trailing slash)", "path", fullRelPath+"/", "pattern", pattern) + skip = true + break + } + } + } + if skip { + continue + } + + if info.IsDir() { + if err := os.MkdirAll(dstPath, info.Mode()); err != nil { + return fmt.Errorf("creating directory %q: %w", dstPath, err) + } + // Recurse with updated prefix. + if err := copyDirRecursiveWithPrefix(srcPath, dstPath, globalBase, fullRelPath, excluded); err != nil { + return err + } + } else { + if err := copyFile(srcPath, dstPath); err != nil { + return err + } + } + } + return nil +} + // getMatchesForPattern returns files/directories matching a pattern relative to sourceDir. // If no matches are found, it logs a debug message and returns an empty slice. // When the pattern ends with "/*", it retries with a recursive "/**" variant. @@ -210,33 +305,59 @@ func copyToTargetWithPatterns( return nil } for file := range filesToCopy { + // Retrieve file information early so that we can adjust exclusion checks if this is a directory. + info, err := os.Stat(file) + if err != nil { + return fmt.Errorf("stating file %q: %w", file, err) + } relPath, err := filepath.Rel(sourceDir, file) if err != nil { return fmt.Errorf("computing relative path for %q: %w", file, err) } relPath = filepath.ToSlash(relPath) skip := false + // For directories, check both the plain relative path and with a trailing slash. for _, ex := range s.ExcludedPaths { - matched, err := u.PathMatch(ex, relPath) - if err != nil { - l.Debug("Error matching exclusion pattern", "pattern", ex, "path", relPath, "error", err) - continue - } else if matched { - l.Debug("Excluding file due to exclusion pattern", "file", relPath, "pattern", ex) - skip = true - break + if info.IsDir() { + matched, err := u.PathMatch(ex, relPath) + if err != nil { + l.Debug("Error matching exclusion pattern", "pattern", ex, "path", relPath, "error", err) + } else if matched { + l.Debug("Excluding directory due to exclusion pattern (plain match)", "directory", relPath, "pattern", ex) + skip = true + break + } + // Also try matching with a trailing slash. + matched, err = u.PathMatch(ex, relPath+"/") + if err != nil { + l.Debug("Error matching exclusion pattern with trailing slash", "pattern", ex, "path", relPath+"/", "error", err) + } else if matched { + l.Debug("Excluding directory due to exclusion pattern (with trailing slash)", "directory", relPath+"/", "pattern", ex) + skip = true + break + } + } else { + // For files, just check the plain relative path. + matched, err := u.PathMatch(ex, relPath) + if err != nil { + l.Debug("Error matching exclusion pattern", "pattern", ex, "path", relPath, "error", err) + } else if matched { + l.Debug("Excluding file due to exclusion pattern", "file", relPath, "pattern", ex) + skip = true + break + } } } if skip { continue } + + // Build the destination path. dstPath := filepath.Join(targetPath, relPath) - info, err := os.Stat(file) - if err != nil { - return fmt.Errorf("stating file %q: %w", file, err) - } if info.IsDir() { - if err := copyDirRecursive(file, dstPath, file, s.ExcludedPaths, nil); err != nil { + // Instead of resetting the base for relative paths, + // use the new recursive function that preserves the global relative path. + if err := copyDirRecursiveWithPrefix(file, dstPath, sourceDir, relPath, s.ExcludedPaths); err != nil { return err } } else { From 1fea6edb5a32a7344540f348416f7953ba0408f3 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Sun, 9 Feb 2025 01:06:06 +0500 Subject: [PATCH 19/20] shallow copy --- internal/exec/copy_glob.go | 66 ++++++++++++--------- tests/fixtures/scenarios/vendor/vendor.yaml | 13 ++++ tests/test-cases/demo-stacks.yaml | 7 +++ 3 files changed, 59 insertions(+), 27 deletions(-) diff --git a/internal/exec/copy_glob.go b/internal/exec/copy_glob.go index 105fa50bd..5528d9303 100644 --- a/internal/exec/copy_glob.go +++ b/internal/exec/copy_glob.go @@ -234,7 +234,7 @@ func copyDirRecursiveWithPrefix(srcDir, dstDir, globalBase, prefix string, exclu // getMatchesForPattern returns files/directories matching a pattern relative to sourceDir. // If no matches are found, it logs a debug message and returns an empty slice. -// When the pattern ends with "/*", it retries with a recursive "/**" variant. +// For patterns ending with "/*" (shallow copy indicator) the function does not fallback to a recursive variant. func getMatchesForPattern(sourceDir, pattern string) ([]string, error) { fullPattern := filepath.Join(sourceDir, pattern) matches, err := u.GetGlobMatches(fullPattern) @@ -242,6 +242,12 @@ func getMatchesForPattern(sourceDir, pattern string) ([]string, error) { return nil, fmt.Errorf("error getting glob matches for %q: %w", fullPattern, err) } if len(matches) == 0 { + // If the pattern ends with "/*" (and not "/**"), do not fallback. + if strings.HasSuffix(pattern, "/*") && !strings.HasSuffix(pattern, "/**") { + l.Debug("No matches found for shallow pattern; target directory will be empty", "pattern", fullPattern) + return []string{}, nil + } + // Fallback for patterns ending with "/*" (non-shallow) or others. if strings.HasSuffix(pattern, "/*") { recursivePattern := strings.TrimSuffix(pattern, "/*") + "/**" fullRecursivePattern := filepath.Join(sourceDir, recursivePattern) @@ -287,25 +293,25 @@ func copyToTargetWithPatterns( return cp.Copy(sourceDir, targetPath) } - // If inclusion patterns are provided, use them to determine which files to copy. - if len(s.IncludedPaths) > 0 { - filesToCopy := make(map[string]struct{}) - for _, pattern := range s.IncludedPaths { - matches, err := getMatchesForPattern(sourceDir, pattern) - if err != nil { - l.Debug("Warning: error getting matches for pattern", "pattern", pattern, "error", err) - continue - } - for _, match := range matches { - filesToCopy[match] = struct{}{} - } + // If inclusion patterns are provided, process each pattern individually. + for _, pattern := range s.IncludedPaths { + // Determine if the pattern indicates shallow copy. + shallow := false + if strings.HasSuffix(pattern, "/*") && !strings.HasSuffix(pattern, "/**") { + shallow = true + } + + matches, err := getMatchesForPattern(sourceDir, pattern) + if err != nil { + l.Debug("Warning: error getting matches for pattern", "pattern", pattern, "error", err) + continue } - if len(filesToCopy) == 0 { - l.Debug("No files matched the inclusion patterns - target directory will be empty") - return nil + if len(matches) == 0 { + l.Debug("No files matched the inclusion pattern", "pattern", pattern) + continue } - for file := range filesToCopy { - // Retrieve file information early so that we can adjust exclusion checks if this is a directory. + for _, file := range matches { + // Retrieve file information. info, err := os.Stat(file) if err != nil { return fmt.Errorf("stating file %q: %w", file, err) @@ -315,8 +321,9 @@ func copyToTargetWithPatterns( return fmt.Errorf("computing relative path for %q: %w", file, err) } relPath = filepath.ToSlash(relPath) + + // Check exclusion patterns (for directories, try both plain and trailing slash). skip := false - // For directories, check both the plain relative path and with a trailing slash. for _, ex := range s.ExcludedPaths { if info.IsDir() { matched, err := u.PathMatch(ex, relPath) @@ -327,7 +334,6 @@ func copyToTargetWithPatterns( skip = true break } - // Also try matching with a trailing slash. matched, err = u.PathMatch(ex, relPath+"/") if err != nil { l.Debug("Error matching exclusion pattern with trailing slash", "pattern", ex, "path", relPath+"/", "error", err) @@ -337,7 +343,6 @@ func copyToTargetWithPatterns( break } } else { - // For files, just check the plain relative path. matched, err := u.PathMatch(ex, relPath) if err != nil { l.Debug("Error matching exclusion pattern", "pattern", ex, "path", relPath, "error", err) @@ -355,10 +360,14 @@ func copyToTargetWithPatterns( // Build the destination path. dstPath := filepath.Join(targetPath, relPath) if info.IsDir() { - // Instead of resetting the base for relative paths, - // use the new recursive function that preserves the global relative path. - if err := copyDirRecursiveWithPrefix(file, dstPath, sourceDir, relPath, s.ExcludedPaths); err != nil { - return err + if shallow { + // Use shallow copy: copy only immediate file entries. + l.Debug("Directory is not copied becasue it is a shallow copy", "directory", relPath) + } else { + // Use the existing recursive copy with prefix. + if err := copyDirRecursiveWithPrefix(file, dstPath, sourceDir, relPath, s.ExcludedPaths); err != nil { + return err + } } } else { if err := copyFile(file, dstPath); err != nil { @@ -366,8 +375,11 @@ func copyToTargetWithPatterns( } } } - } else { - // No inclusion patterns defined; copy everything except those matching excluded items. + } + + // If no inclusion patterns are defined; copy everything except those matching excluded items. + // (This branch is preserved from the original logic.) + if len(s.IncludedPaths) == 0 { if err := copyDirRecursive(sourceDir, targetPath, sourceDir, s.ExcludedPaths, s.IncludedPaths); err != nil { return fmt.Errorf("error copying from %q to %q: %w", sourceDir, targetPath, err) } diff --git a/tests/fixtures/scenarios/vendor/vendor.yaml b/tests/fixtures/scenarios/vendor/vendor.yaml index ba3c016f5..0d641f09d 100644 --- a/tests/fixtures/scenarios/vendor/vendor.yaml +++ b/tests/fixtures/scenarios/vendor/vendor.yaml @@ -72,3 +72,16 @@ spec: - "components/library/" tags: - demo + + - component: "test shallow globs and folder exclusion" + source: "github.com/cloudposse/atmos.git" + included_paths: + - "**/demo-localstack/*" + - "**/demo-library/**" + excluded_paths: + - "**/demo-library/**/stargazers/**" + - "**/demo-library/**/*.tf" + targets: + - "components/globs/" + tags: + - demo diff --git a/tests/test-cases/demo-stacks.yaml b/tests/test-cases/demo-stacks.yaml index 367742db9..28ca2b165 100644 --- a/tests/test-cases/demo-stacks.yaml +++ b/tests/test-cases/demo-stacks.yaml @@ -193,4 +193,11 @@ tests: - "./components/library/examples/demo-stacks/components/terraform/myapp/variables.tf" - "./components/library/examples/demo-stacks/components/terraform/myapp/versions.tf" - "./components/library/weather/README.md" + - "./components/globs/examples/demo-library/ipinfo/README.md" + - "./components/globs/examples/demo-library/weather/README.md" + - "./components/globs/examples/demo-library/README.md" + - "./components/globs/examples/demo-localstack/.gitignore" + - "./components/globs/examples/demo-localstack/atmos.yaml" + - "./components/globs/examples/demo-localstack/docker-compose.yml" + - "./components/globs/examples/demo-localstack/README.md" exit_code: 0 From 618860d12ef3eaee28fc65e3ff4f92dada754d7e Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Mon, 10 Feb 2025 16:13:35 +0500 Subject: [PATCH 20/20] docs added --- .../core-concepts/vendor/vendor-manifest.mdx | 90 ++++++++++++++++++- 1 file changed, 89 insertions(+), 1 deletion(-) diff --git a/website/docs/core-concepts/vendor/vendor-manifest.mdx b/website/docs/core-concepts/vendor/vendor-manifest.mdx index c41e2fec0..ecaa6af4b 100644 --- a/website/docs/core-concepts/vendor/vendor-manifest.mdx +++ b/website/docs/core-concepts/vendor/vendor-manifest.mdx @@ -217,7 +217,7 @@ The `vendor.yaml` vendoring manifest supports Kubernetes-style YAML config to de
`included_paths` and `excluded_paths`
- `included_paths` and `excluded_paths` support [POSIX-style greedy Globs](https://en.wikipedia.org/wiki/Glob_(programming)) for filenames/paths (double-star/globstar `**` is supported as well). + `included_paths` and `excluded_paths` support [POSIX-style greedy Globs](https://en.wikipedia.org/wiki/Glob_(programming)) for filenames/paths (double-star/globstar `**` is supported as well). For more details, see [Vendoring with Globs](#vendoring-with-globs).
`component`
@@ -497,3 +497,91 @@ To vendor the `vpc` component, execute the following command: atmos vendor pull -c vpc ``` + +## Vendoring with Globs + +When defining vendoring rules in Atmos, **glob patterns** determine which files and directories are included or excluded. Understanding how globs behave—especially when using greedy (`**`) versus non-greedy (`*`) patterns—is crucial for precise vendoring. + +### Understanding Greedy and Non-Greedy Globs + +Globs use special wildcard characters: + +- `*` (single asterisk) matches any sequence of characters **within a single path segment**. +- `**` (double asterisk) matches across multiple path segments **recursively**. + +This distinction is important when excluding specific directories or files while vendoring. + +#### Example: Excluding a Subdirectory + +Consider the following configuration: + +```yaml +included_paths: + - "**/demo-library/**" +excluded_paths: + - "**/demo-library/**/stargazers/**" +``` + +How It Works: +- The included_paths rule `**/demo-library/**` ensures all files inside `demo-library` (at any depth) are vendored. +- The excluded_paths rule `**/demo-library/**/stargazers/**` prevents any files inside `stargazers` subdirectories from being vendored. + +This means: +- All files within demo-library except those inside any `stargazers` subdirectory are vendored. +- Any other files outside `stargazers` are unaffected by this exclusion. + +Example: A Non-Recursive Pattern That Doesn't Work + +```yaml +included_paths: + - "**/demo-library/*" +excluded_paths: + - "**/demo-library/**/stargazers/**" +``` + +In this case: +- `**/demo-library/*` only matches immediate children of demo-library, not nested files or subdirectories. +- This means `stargazers/` itself could be matched, but its contents might not be explicitly excluded. +- To correctly capture all subdirectories and files while still excluding stargazers, use `**/demo-library/**/*`. + +Using `{...}` for Multiple Extensions or Patterns + +Curly braces `{...}` allow for expanding multiple patterns into separate glob matches. This is useful when selecting multiple file types or directories within a single glob pattern. + +Example: Matching Multiple File Extensions + +```yaml +included_paths: + - "**/demo-library/**/*.{tf,md}" +``` + +This is equivalent to writing: + +```yaml +included_paths: + - "**/demo-library/**/*.tf" + - "**/demo-library/**/*.md" +``` + +The `{tf,md}` part expands to both `*.tf` and `*.md`, making the rule more concise. + +Example: Excluding Multiple Directories + +```yaml +excluded_paths: + - "**/demo-library/**/{stargazers,archive}/**" +``` + +This excludes both: +- `**/demo-library/**/stargazers/**` +- `**/demo-library/**/archive/**` + +Using `{...}` here prevents the need to write two separate exclusion rules. + +Key Takeaways + 1. Use `**/` for recursive matching to include everything inside a directory. + 2. Use `*` for single-segment matches, which won't include deeper subdirectories. + 3. Use `{...}` to match multiple extensions or directories within a single pattern. + 4. Exclusion rules must match nested paths explicitly when trying to exclude deep directories. + +By carefully combining `included_paths`, `excluded_paths`, and `{...}` expansion, you can precisely control which files are vendored while ensuring unwanted directories like stargazers are omitted. \ No newline at end of file