diff --git a/go.mod b/go.mod index 86f4fe9ad8..713717d7f0 100644 --- a/go.mod +++ b/go.mod @@ -91,6 +91,7 @@ require ( github.com/Shopify/ejson v1.3.3 // indirect github.com/a8m/envsubst v1.4.2 // indirect github.com/agext/levenshtein v1.2.2 // indirect + github.com/agiledragon/gomonkey/v2 v2.13.0 github.com/agnivade/levenshtein v1.2.1 // indirect github.com/alecthomas/chroma/v2 v2.14.0 // indirect github.com/alecthomas/participle/v2 v2.1.1 // indirect diff --git a/go.sum b/go.sum index a1a957c4de..8bcd9599a3 100644 --- a/go.sum +++ b/go.sum @@ -706,6 +706,8 @@ github.com/a8m/envsubst v1.4.2 h1:4yWIHXOLEJHQEFd4UjrWDrYeYlV7ncFWJOCBRLOZHQg= github.com/a8m/envsubst v1.4.2/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY= github.com/agext/levenshtein v1.2.2 h1:0S/Yg6LYmFJ5stwQeRp6EeOcCbj7xiqQSdNelsXvaqE= github.com/agext/levenshtein v1.2.2/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= +github.com/agiledragon/gomonkey/v2 v2.13.0 h1:B24Jg6wBI1iB8EFR1c+/aoTg7QN/Cum7YffG8KMIyYo= +github.com/agiledragon/gomonkey/v2 v2.13.0/go.mod h1:ap1AmDzcVOAz1YpeJ3TCzIgstoaWLA6jbbgxfB4w2iY= github.com/agnivade/levenshtein v1.2.1 h1:EHBY3UOn1gwdy/VbFwgo4cxecRznFk7fKWN1KOX7eoM= github.com/agnivade/levenshtein v1.2.1/go.mod h1:QVVI16kDrtSuwcpd0p1+xMC6Z/VfhtCyDIjcwga4/DU= github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY= @@ -1290,6 +1292,7 @@ github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+ github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0= github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= @@ -1770,6 +1773,7 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8= github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= @@ -2339,6 +2343,7 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= diff --git a/internal/exec/copy_glob.go b/internal/exec/copy_glob.go new file mode 100644 index 0000000000..93854a9b7e --- /dev/null +++ b/internal/exec/copy_glob.go @@ -0,0 +1,475 @@ +package exec + +import ( + "fmt" + "io" + "os" + "path/filepath" + "strings" + + log "github.com/charmbracelet/log" + "github.com/cloudposse/atmos/pkg/schema" + u "github.com/cloudposse/atmos/pkg/utils" + cp "github.com/otiai10/copy" // Using the optimized copy library when no filtering is required. +) + +// Named constants to avoid literal duplication. +const ( + logKeyPath = "path" + logKeyError = "error" + logKeyPattern = "pattern" + shallowCopySuffix = "/*" + + // finalTargetKey is used as a logging key for the final target. + finalTargetKey = "finalTarget" + + // sourceKey is used as a logging key for the source. + sourceKey = "source" +) + +// PrefixCopyContext groups parameters for prefix-based copy operations. +type PrefixCopyContext struct { + SrcDir string + DstDir string + GlobalBase string + Prefix string + Excluded []string +} + +// CopyContext groups parameters for directory copy operations. +type CopyContext struct { + SrcDir string + DstDir string + BaseDir string + Excluded []string + Included []string +} + +// copyFile copies a single file from src to dst while preserving file permissions. +func copyFile(src, dst string) error { + sourceFile, err := os.Open(src) + if err != nil { + return fmt.Errorf("opening source file %q: %w", src, err) + } + defer sourceFile.Close() + + if err := os.MkdirAll(filepath.Dir(dst), os.ModePerm); err != nil { + return fmt.Errorf("creating destination directory for %q: %w", dst, err) + } + + destinationFile, err := os.Create(dst) + if err != nil { + return fmt.Errorf("creating destination file %q: %w", dst, err) + } + defer destinationFile.Close() + + if _, err := io.Copy(destinationFile, sourceFile); err != nil { + return fmt.Errorf("copying content from %q to %q: %w", src, dst, err) + } + + info, err := os.Stat(src) + if err != nil { + return fmt.Errorf("getting file info for %q: %w", src, err) + } + if err := os.Chmod(dst, info.Mode()); err != nil { + return fmt.Errorf("setting permissions on %q: %w", dst, err) + } + return nil +} + +// shouldExcludePath checks exclusion patterns for a given relative path and file info. +func shouldExcludePath(info os.FileInfo, relPath string, excluded []string) bool { + for _, pattern := range excluded { + // Check plain relative path. + matched, err := u.PathMatch(pattern, relPath) + if err != nil { + log.Debug("Error matching exclusion pattern", logKeyPath, relPath, logKeyError, err) + continue + } + if matched { + log.Debug("Excluding path due to exclusion pattern (plain match)", logKeyPath, relPath, logKeyPattern, pattern) + return true + } + // If a directory, also check with a trailing slash. + if info.IsDir() { + matched, err = u.PathMatch(pattern, relPath+"/") + if err != nil { + log.Debug("Error matching exclusion pattern with trailing slash", logKeyPattern, pattern, logKeyPath, relPath+"/", logKeyError, err) + continue + } + if matched { + log.Debug("Excluding directory due to exclusion pattern (with trailing slash)", logKeyPath, relPath+"/", logKeyPattern, pattern) + return true + } + } + } + return false +} + +// shouldIncludePath checks whether a file should be included based on inclusion patterns. +func shouldIncludePath(info os.FileInfo, relPath string, included []string) bool { + // Directories are generally handled by recursion. + if len(included) == 0 || info.IsDir() { + return true + } + for _, pattern := range included { + matched, err := u.PathMatch(pattern, relPath) + if err != nil { + log.Debug("Error matching inclusion pattern", logKeyPattern, pattern, logKeyPath, relPath, logKeyError, err) + continue + } + if matched { + log.Debug("Including path due to inclusion pattern", logKeyPath, relPath, logKeyPattern, pattern) + return true + } + } + log.Debug("Excluding path because it does not match any inclusion pattern", logKeyPath, relPath) + return false +} + +// shouldSkipEntry determines whether to skip a file/directory based on its relative path to baseDir. +func shouldSkipEntry(info os.FileInfo, srcPath, baseDir string, excluded, included []string) bool { + if info.Name() == ".git" { + return true + } + relPath, err := filepath.Rel(baseDir, srcPath) + if err != nil { + log.Debug("Error computing relative path", "srcPath", srcPath, logKeyError, err) + return true // treat error as a signal to skip + } + relPath = filepath.ToSlash(relPath) + if shouldExcludePath(info, relPath, excluded) { + return true + } + if !shouldIncludePath(info, relPath, included) { + return true + } + return false +} + +// processDirEntry handles a single directory entry for copyDirRecursive. +func processDirEntry(entry os.DirEntry, ctx *CopyContext) error { + srcPath := filepath.Join(ctx.SrcDir, entry.Name()) + dstPath := filepath.Join(ctx.DstDir, entry.Name()) + + info, err := entry.Info() + if err != nil { + return fmt.Errorf("getting info for %q: %w", srcPath, err) + } + + if shouldSkipEntry(info, srcPath, ctx.BaseDir, ctx.Excluded, ctx.Included) { + log.Debug("Skipping entry", "srcPath", srcPath) + return nil + } + + // Skip symlinks. + if info.Mode()&os.ModeSymlink != 0 { + log.Debug("Skipping symlink", logKeyPath, srcPath) + return nil + } + + if info.IsDir() { + if err := os.MkdirAll(dstPath, info.Mode()); err != nil { + return fmt.Errorf("creating directory %q: %w", dstPath, err) + } + // Recurse with the same context but with updated source and destination directories. + newCtx := &CopyContext{ + SrcDir: srcPath, + DstDir: dstPath, + BaseDir: ctx.BaseDir, + Excluded: ctx.Excluded, + Included: ctx.Included, + } + return copyDirRecursive(newCtx) + } + return copyFile(srcPath, dstPath) +} + +// copyDirRecursive recursively copies srcDir to dstDir using shouldSkipEntry filtering. +func copyDirRecursive(ctx *CopyContext) error { + entries, err := os.ReadDir(ctx.SrcDir) + if err != nil { + return fmt.Errorf("reading directory %q: %w", ctx.SrcDir, err) + } + for _, entry := range entries { + if err := processDirEntry(entry, ctx); err != nil { + return err + } + } + return nil +} + +// shouldSkipPrefixEntry checks exclusion patterns for copyDirRecursiveWithPrefix. +func shouldSkipPrefixEntry(info os.FileInfo, fullRelPath string, excluded []string) bool { + for _, pattern := range excluded { + matched, err := u.PathMatch(pattern, fullRelPath) + if err != nil { + log.Debug("Error matching exclusion pattern in prefix function", logKeyPattern, pattern, logKeyPath, fullRelPath, logKeyError, err) + continue + } + if matched { + log.Debug("Excluding (prefix) due to exclusion pattern (plain match)", logKeyPath, fullRelPath, logKeyPattern, pattern) + return true + } + if info.IsDir() { + matched, err = u.PathMatch(pattern, fullRelPath+"/") + if err != nil { + log.Debug("Error matching exclusion pattern with trailing slash in prefix function", logKeyPattern, pattern, logKeyPath, fullRelPath+"/", logKeyError, err) + continue + } + if matched { + log.Debug("Excluding (prefix) due to exclusion pattern (with trailing slash)", logKeyPath, fullRelPath+"/", logKeyPattern, pattern) + return true + } + } + } + return false +} + +// processPrefixEntry handles a single entry for copyDirRecursiveWithPrefix. +func processPrefixEntry(entry os.DirEntry, ctx *PrefixCopyContext) error { + fullRelPath := filepath.ToSlash(filepath.Join(ctx.Prefix, entry.Name())) + srcPath := filepath.Join(ctx.SrcDir, entry.Name()) + dstPath := filepath.Join(ctx.DstDir, entry.Name()) + + info, err := entry.Info() + if err != nil { + return fmt.Errorf("getting info for %q: %w", srcPath, err) + } + + if entry.Name() == ".git" { + log.Debug("Skipping .git directory", logKeyPath, fullRelPath) + return nil + } + + if shouldSkipPrefixEntry(info, fullRelPath, ctx.Excluded) { + return nil + } + + if info.IsDir() { + if err := os.MkdirAll(dstPath, info.Mode()); err != nil { + return fmt.Errorf("creating directory %q: %w", dstPath, err) + } + newCtx := &PrefixCopyContext{ + SrcDir: srcPath, + DstDir: dstPath, + GlobalBase: ctx.GlobalBase, + Prefix: fullRelPath, + Excluded: ctx.Excluded, + } + return copyDirRecursiveWithPrefix(newCtx) + } + return copyFile(srcPath, dstPath) +} + +// copyDirRecursiveWithPrefix recursively copies srcDir to dstDir while preserving the global relative path. +func copyDirRecursiveWithPrefix(ctx *PrefixCopyContext) error { + entries, err := os.ReadDir(ctx.SrcDir) + if err != nil { + return fmt.Errorf("reading directory %q: %w", ctx.SrcDir, err) + } + for _, entry := range entries { + if err := processPrefixEntry(entry, ctx); err != nil { + return err + } + } + return nil +} + +// getMatchesForPattern returns files/directories matching a pattern relative to sourceDir. +func getMatchesForPattern(sourceDir, pattern string) ([]string, error) { + fullPattern := filepath.Join(sourceDir, pattern) + matches, err := u.GetGlobMatches(fullPattern) + if err != nil { + return nil, fmt.Errorf("error getting glob matches for %q: %w", fullPattern, err) + } + if len(matches) != 0 { + return matches, nil + } + + // Handle shallow copy indicator. + if strings.HasSuffix(pattern, shallowCopySuffix) { + if !strings.HasSuffix(pattern, "/**") { + log.Debug("No matches found for shallow pattern; target directory will be empty", logKeyPattern, fullPattern) + return []string{}, nil + } + recursivePattern := strings.TrimSuffix(pattern, shallowCopySuffix) + "/**" + fullRecursivePattern := filepath.Join(sourceDir, recursivePattern) + matches, err = u.GetGlobMatches(fullRecursivePattern) + if err != nil { + return nil, fmt.Errorf("error getting glob matches for recursive pattern %q: %w", fullRecursivePattern, err) + } + if len(matches) == 0 { + log.Debug("No matches found for recursive pattern; target directory will be empty", logKeyPattern, fullRecursivePattern) + return []string{}, nil + } + return matches, nil + } + + log.Debug("No matches found for pattern; target directory will be empty", logKeyPattern, fullPattern) + return []string{}, nil +} + +// isShallowPattern determines if a pattern indicates a shallow copy. +func isShallowPattern(pattern string) bool { + return strings.HasSuffix(pattern, shallowCopySuffix) && !strings.HasSuffix(pattern, "/**") +} + +// processMatch handles a single file/directory match for copyToTargetWithPatterns. +func processMatch(sourceDir, targetPath, file string, shallow bool, excluded []string) error { + info, err := os.Stat(file) + if err != nil { + return fmt.Errorf("stating file %q: %w", file, err) + } + relPath, err := filepath.Rel(sourceDir, file) + if err != nil { + return fmt.Errorf("computing relative path for %q: %w", file, err) + } + relPath = filepath.ToSlash(relPath) + if shouldExcludePath(info, relPath, excluded) { + return nil + } + + dstPath := filepath.Join(targetPath, relPath) + if info.IsDir() { + if shallow { + log.Debug("Directory is not copied because it is a shallow copy", "directory", relPath) + return nil + } + return copyDirRecursiveWithPrefix(&PrefixCopyContext{ + SrcDir: file, + DstDir: dstPath, + GlobalBase: sourceDir, + Prefix: relPath, + Excluded: excluded, + }) + } + return copyFile(file, dstPath) +} + +// processIncludedPattern handles all matches for one inclusion pattern. +func processIncludedPattern(sourceDir, targetPath, pattern string, excluded []string) error { + shallow := isShallowPattern(pattern) + matches, err := getMatchesForPattern(sourceDir, pattern) + if err != nil { + log.Debug("Warning: error getting matches for pattern", logKeyPattern, pattern, logKeyError, err) + return nil + } + if len(matches) == 0 { + log.Debug("No files matched the inclusion pattern", logKeyPattern, pattern) + return nil + } + for _, file := range matches { + if err := processMatch(sourceDir, targetPath, file, shallow, excluded); err != nil { + return err + } + } + return nil +} + +// initFinalTarget initializes the final target path based on source type. +func initFinalTarget(sourceDir, targetPath string, sourceIsLocalFile bool) (string, error) { + if sourceIsLocalFile { + return getLocalFinalTarget(sourceDir, targetPath) + } + return getNonLocalFinalTarget(targetPath) +} + +func getLocalFinalTarget(sourceDir, targetPath string) (string, error) { + if filepath.Ext(targetPath) == "" { + if err := os.MkdirAll(targetPath, os.ModePerm); err != nil { + return "", fmt.Errorf("creating target directory %q: %w", targetPath, err) + } + return filepath.Join(targetPath, SanitizeFileName(filepath.Base(sourceDir))), nil + } + + parent := filepath.Dir(targetPath) + if err := os.MkdirAll(parent, os.ModePerm); err != nil { + return "", fmt.Errorf("creating parent directory %q: %w", parent, err) + } + return targetPath, nil +} + +func getNonLocalFinalTarget(targetPath string) (string, error) { + if err := os.MkdirAll(targetPath, os.ModePerm); err != nil { + return "", fmt.Errorf("creating target directory %q: %w", targetPath, err) + } + return targetPath, nil +} + +// handleLocalFileSource handles copy for local file sources. +func handleLocalFileSource(sourceDir, finalTarget string) error { + log.Debug("Local file source detected; invoking ComponentOrMixinsCopy", + "sourceFile", sourceDir, finalTargetKey, finalTarget) + return ComponentOrMixinsCopy(sourceDir, finalTarget) +} + +// copyToTargetWithPatterns copies the contents from sourceDir to targetPath, applying inclusion and exclusion patterns. +func copyToTargetWithPatterns( + sourceDir, targetPath string, + s *schema.AtmosVendorSource, + sourceIsLocalFile bool, +) error { + finalTarget, err := initFinalTarget(sourceDir, targetPath, sourceIsLocalFile) + if err != nil { + return err + } + log.Debug("Copying files", sourceKey, sourceDir, finalTargetKey, finalTarget) + if sourceIsLocalFile { + return handleLocalFileSource(sourceDir, finalTarget) + } + // If no inclusion or exclusion patterns are defined, use the cp library. + if len(s.IncludedPaths) == 0 && len(s.ExcludedPaths) == 0 { + log.Debug("No inclusion or exclusion patterns defined; using cp.Copy for fast copy", "source", sourceDir, finalTargetKey, finalTarget) + return cp.Copy(sourceDir, finalTarget) + } + // Process each inclusion pattern. + for _, pattern := range s.IncludedPaths { + log.Debug("Processing inclusion pattern", "pattern", pattern, "source", sourceDir, finalTargetKey, finalTarget) + if err := processIncludedPattern(sourceDir, finalTarget, pattern, s.ExcludedPaths); err != nil { + return err + } + } + // Copy entire directory if no inclusion patterns are defined. + if len(s.IncludedPaths) == 0 { + log.Debug("No inclusion patterns defined; copying entire directory recursively", "source", sourceDir, finalTargetKey, finalTarget) + if err := copyDirRecursive(&CopyContext{ + SrcDir: sourceDir, + DstDir: finalTarget, + BaseDir: sourceDir, + Excluded: s.ExcludedPaths, + Included: s.IncludedPaths, + }); err != nil { + return fmt.Errorf("error copying from %q to %q: %w", sourceDir, finalTarget, err) + } + } + return nil +} + +// ComponentOrMixinsCopy covers 2 cases: file-to-folder and file-to-file copy. +func ComponentOrMixinsCopy(sourceFile, finalTarget string) error { + var dest string + if filepath.Ext(finalTarget) == "" { + // File-to-folder copy: append the source file's base name to the directory. + dest = filepath.Join(finalTarget, filepath.Base(sourceFile)) + log.Debug("ComponentOrMixinsCopy: file-to-folder copy", "sourceFile", sourceFile, "destination", dest) + } else { + // File-to-file copy: use finalTarget as is. + dest = finalTarget + // Create only the parent directory. + parent := filepath.Dir(dest) + if err := os.MkdirAll(parent, os.ModePerm); err != nil { + log.Debug("ComponentOrMixinsCopy: error creating parent directory", "parent", parent, "error", err) + return fmt.Errorf("creating parent directory %q: %w", parent, err) + } + log.Debug("ComponentOrMixinsCopy: file-to-file copy", "sourceFile", sourceFile, "destination", dest) + } + // Remove any existing directory at dest to avoid "is a directory" errors. + if info, err := os.Stat(dest); err == nil && info.IsDir() { + log.Debug("ComponentOrMixinsCopy: destination exists as directory, removing", "destination", dest) + if err := os.RemoveAll(dest); err != nil { + return fmt.Errorf("removing existing directory %q: %w", dest, err) + } + } + return cp.Copy(sourceFile, dest) +} diff --git a/internal/exec/copy_glob_test.go b/internal/exec/copy_glob_test.go new file mode 100644 index 0000000000..f02addf06a --- /dev/null +++ b/internal/exec/copy_glob_test.go @@ -0,0 +1,970 @@ +package exec + +import ( + "errors" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + cp "github.com/otiai10/copy" + + "github.com/cloudposse/atmos/pkg/schema" + "github.com/cloudposse/atmos/pkg/utils" + u "github.com/cloudposse/atmos/pkg/utils" +) + +var ( + errSimulatedChmodFailure = errors.New("simulated chmod failure") + errSimulatedGlobError = errors.New("simulated glob error") + errForcedInfoError = errors.New("forced info error") + errSimulatedMkdirAllError = errors.New("simulated MkdirAll error") + errSimulatedRelPathError = errors.New("simulated relative path error") +) + +// Use a local variable to override the glob matching function in tests. +var getGlobMatchesForTest = utils.GetGlobMatches + +// Helper that calls our local getGlobMatchesForTest. +func getMatchesForPatternForTest(sourceDir, pattern string) ([]string, error) { + fullPattern := filepath.Join(sourceDir, pattern) + // Normalize fullPattern to use forward slashes. + normalized := filepath.ToSlash(fullPattern) + return getGlobMatchesForTest(normalized) +} + +type fakeDirEntry struct { + name string + err error +} + +func (f fakeDirEntry) Name() string { return f.name } +func (f fakeDirEntry) IsDir() bool { return false } +func (f fakeDirEntry) Type() os.FileMode { return 0 } +func (f fakeDirEntry) Info() (os.FileInfo, error) { return nil, f.err } + +// TestCopyFile verifies that copyFile correctly copies file contents and preserves permissions. +func TestCopyFile(t *testing.T) { + srcDir, err := os.MkdirTemp("", "copyfile-src") + if err != nil { + t.Fatalf("Failed to create source dir: %v", err) + } + defer os.RemoveAll(srcDir) + dstDir, err := os.MkdirTemp("", "copyfile-dst") + if err != nil { + t.Fatalf("Failed to create destination dir: %v", err) + } + defer os.RemoveAll(dstDir) + srcFile := filepath.Join(srcDir, "test.txt") + content := "copyFileTest" + if err := os.WriteFile(srcFile, []byte(content), 0o600); err != nil { + t.Fatalf("Failed to write source file: %v", err) + } + dstFile := filepath.Join(dstDir, "test.txt") + if err := copyFile(srcFile, dstFile); err != nil { + t.Fatalf("copyFile failed: %v", err) + } + copiedContent, err := os.ReadFile(dstFile) + if err != nil { + t.Fatalf("Failed to read destination file: %v", err) + } + if string(copiedContent) != content { + t.Errorf("Expected content %q, got %q", content, string(copiedContent)) + } +} + +// TestCopyFile_SourceNotExist tests error in copyFile when source file does not exist. +func TestCopyFile_SourceNotExist(t *testing.T) { + nonExistent := filepath.Join(os.TempDir(), "nonexistent.txt") + dstFile := filepath.Join(os.TempDir(), "dst.txt") + err := copyFile(nonExistent, dstFile) + if err == nil || !strings.Contains(err.Error(), "opening source file") { + t.Errorf("Expected error for non-existent source file, got %v", err) + } +} + +// TestShouldExcludePath checks that a file is excluded by pattern. +func TestShouldExcludePath(t *testing.T) { + tmpFile, err := os.CreateTemp("", "test.log") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer os.Remove(tmpFile.Name()) + info, err := tmpFile.Stat() + if err != nil { + t.Fatalf("Failed to stat file: %v", err) + } + excluded := []string{"**/*.log"} + if !shouldExcludePath(info, "app/test.log", excluded) { + t.Errorf("Expected path to be excluded") + } +} + +// TestShouldExcludePath_Directory checks directory exclusion using a trailing slash. +// Skipped on Windows. +func TestShouldExcludePath_Directory(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Skipping directory exclusion test on Windows") + } + dir, err := os.MkdirTemp("", "dir-exclude") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(dir) + info, err := os.Stat(dir) + if err != nil { + t.Fatalf("Failed to stat directory: %v", err) + } + pattern := "**/" + filepath.Base(dir) + "/" + if !shouldExcludePath(info, filepath.Base(dir), []string{pattern}) { + t.Errorf("Expected directory %q to be excluded by pattern %q", filepath.Base(dir), pattern) + } +} + +// TestShouldExcludePath_Error ensures invalid patterns do not exclude files. +func TestShouldExcludePath_Error(t *testing.T) { + tmpFile, err := os.CreateTemp("", "test.log") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer os.Remove(tmpFile.Name()) + info, err := tmpFile.Stat() + if err != nil { + t.Fatalf("Failed to stat file: %v", err) + } + if shouldExcludePath(info, "app/test.log", []string{"[abc"}) { + t.Errorf("Expected path not to be excluded by invalid pattern") + } +} + +// TestShouldIncludePath checks that a file is included by pattern. +func TestShouldIncludePath(t *testing.T) { + tmpFile, err := os.CreateTemp("", "test.txt") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer os.Remove(tmpFile.Name()) + info, err := tmpFile.Stat() + if err != nil { + t.Fatalf("Failed to stat file: %v", err) + } + included := []string{"**/*.txt"} + if !shouldIncludePath(info, "docs/readme.txt", included) { + t.Errorf("Expected path to be included") + } +} + +// TestShouldIncludePath_Error ensures invalid inclusion patterns do not include files. +func TestShouldIncludePath_Error(t *testing.T) { + tmpFile, err := os.CreateTemp("", "test.txt") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer os.Remove(tmpFile.Name()) + info, err := tmpFile.Stat() + if err != nil { + t.Fatalf("Failed to stat file: %v", err) + } + if shouldIncludePath(info, "docs/readme.txt", []string{"[abc"}) { + t.Errorf("Expected path not to be included by invalid pattern") + } +} + +// TestShouldSkipEntry verifies that a file is skipped if it matches an excluded pattern. +func TestShouldSkipEntry(t *testing.T) { + baseDir, err := os.MkdirTemp("", "base") + if err != nil { + t.Fatalf("Failed to create base dir: %v", err) + } + defer os.RemoveAll(baseDir) + subDir := filepath.Join(baseDir, "sub") + if err := os.Mkdir(subDir, 0o755); err != nil { + t.Fatalf("Failed to create sub dir: %v", err) + } + filePath := filepath.Join(subDir, "sample.txt") + if err := os.WriteFile(filePath, []byte("test"), 0o600); err != nil { + t.Fatalf("Failed to write file: %v", err) + } + info, err := os.Stat(filePath) + if err != nil { + t.Fatalf("Failed to stat file: %v", err) + } + if !shouldSkipEntry(info, filePath, baseDir, []string{"**/*.txt"}, []string{}) { + t.Errorf("Expected file %q to be skipped", filePath) + } +} + +// TestCopyDirRecursive ensures that copyDirRecursive copies a directory tree. +func TestCopyDirRecursive(t *testing.T) { + srcDir, err := os.MkdirTemp("", "copydir-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + dstDir, err := os.MkdirTemp("", "copydir-dst") + if err != nil { + t.Fatalf("Failed to create dst dir: %v", err) + } + defer os.RemoveAll(dstDir) + subDir := filepath.Join(srcDir, "sub") + if err := os.Mkdir(subDir, 0o755); err != nil { + t.Fatalf("Failed to create sub dir: %v", err) + } + file1 := filepath.Join(srcDir, "file1.txt") + file2 := filepath.Join(subDir, "file2.txt") + if err := os.WriteFile(file1, []byte("file1"), 0o600); err != nil { + t.Fatalf("Failed to write file1: %v", err) + } + if err := os.WriteFile(file2, []byte("file2"), 0o600); err != nil { + t.Fatalf("Failed to write file2: %v", err) + } + ctx := &CopyContext{ + SrcDir: srcDir, + DstDir: dstDir, + BaseDir: srcDir, + Excluded: []string{}, + Included: []string{}, + } + if err := copyDirRecursive(ctx); err != nil { + t.Fatalf("copyDirRecursive failed: %v", err) + } + if _, err := os.Stat(filepath.Join(dstDir, "file1.txt")); os.IsNotExist(err) { + t.Errorf("Expected file1.txt to exist") + } + if _, err := os.Stat(filepath.Join(dstDir, "sub", "file2.txt")); os.IsNotExist(err) { + t.Errorf("Expected file2.txt to exist") + } +} + +// TestProcessDirEntry_Symlink ensures that symlink entries are skipped. +func TestProcessDirEntry_Symlink(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Skipping symlink test on Windows") + } + srcDir, err := os.MkdirTemp("", "symlink-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + dstDir, err := os.MkdirTemp("", "symlink-dst") + if err != nil { + t.Fatalf("Failed to create dst dir: %v", err) + } + defer os.RemoveAll(dstDir) + targetFile := filepath.Join(srcDir, "target.txt") + if err := os.WriteFile(targetFile, []byte("data"), 0o600); err != nil { + t.Fatalf("Failed to write target file: %v", err) + } + linkPath := filepath.Join(srcDir, "link.txt") + if err := os.Symlink(targetFile, linkPath); err != nil { + t.Skip("Cannot create symlink on this system, skipping test.") + } + entries, err := os.ReadDir(srcDir) + if err != nil { + t.Fatalf("Failed to read src dir: %v", err) + } + var linkEntry os.DirEntry + for _, e := range entries { + if e.Name() == "link.txt" { + linkEntry = e + break + } + } + if linkEntry == nil { + t.Fatalf("Symlink entry not found") + } + ctx := &CopyContext{ + SrcDir: srcDir, + DstDir: dstDir, + BaseDir: srcDir, + Excluded: []string{}, + Included: []string{}, + } + if err := processDirEntry(linkEntry, ctx); err != nil { + t.Errorf("Expected nil error for symlink, got %v", err) + } + if _, err := os.Stat(filepath.Join(dstDir, "link.txt")); err == nil { + t.Errorf("Expected symlink not to be copied") + } +} + +// TestGetMatchesForPattern checks that getMatchesForPattern returns expected matches. +func TestGetMatchesForPattern(t *testing.T) { + srcDir, err := os.MkdirTemp("", "glob-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + fileA := filepath.Join(srcDir, "a.txt") + fileB := filepath.Join(srcDir, "b.log") + if err := os.WriteFile(fileA, []byte("content"), 0o600); err != nil { + t.Fatalf("Failed to write fileA: %v", err) + } + if err := os.WriteFile(fileB, []byte("content"), 0o600); err != nil { + t.Fatalf("Failed to write fileB: %v", err) + } + matches, err := getMatchesForPattern(srcDir, "*.txt") + if err != nil { + t.Fatalf("getMatchesForPattern error: %v", err) + } + if len(matches) == 0 || !strings.Contains(matches[0], "a.txt") { + t.Errorf("Expected match for a.txt, got %v", matches) + } +} + +// TestGetMatchesForPattern_NoMatches uses our helper to simulate no matches. +func TestGetMatchesForPattern_NoMatches(t *testing.T) { + oldFn := getGlobMatchesForTest + defer func() { getGlobMatchesForTest = oldFn }() + getGlobMatchesForTest = func(pattern string) ([]string, error) { + return []string{}, nil + } + srcDir, err := os.MkdirTemp("", "nomatch-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + matches, err := getMatchesForPatternForTest(srcDir, "nonexistent*") + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if len(matches) != 0 { + t.Errorf("Expected 0 matches, got %v", matches) + } +} + +// TestGetMatchesForPattern_InvalidPattern ensures invalid patterns produce an error. +func TestGetMatchesForPattern_InvalidPattern(t *testing.T) { + srcDir, err := os.MkdirTemp("", "invalid-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + _, err = getMatchesForPattern(srcDir, "[") + if err == nil { + t.Errorf("Expected error for invalid pattern, got nil") + } +} + +// TestGetMatchesForPattern_ShallowNoMatch tests the shallow branch with no matches. +func TestGetMatchesForPattern_ShallowNoMatch(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Skipping shallow no-match test on Windows") + } + oldFn := getGlobMatchesForTest + defer func() { getGlobMatchesForTest = oldFn }() + getGlobMatchesForTest = func(pattern string) ([]string, error) { + normalized := filepath.ToSlash(pattern) + if strings.Contains(normalized, "/*") && !strings.Contains(normalized, "/**") { + return []string{}, nil + } + return []string{}, nil + } + srcDir, err := os.MkdirTemp("", "shallow-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + emptyDir := filepath.Join(srcDir, "dir") + if err := os.Mkdir(emptyDir, 0o755); err != nil { + t.Fatalf("Failed to create empty directory: %v", err) + } + _, err = getMatchesForPatternForTest(srcDir, "dir/*") + if err != nil { + t.Fatalf("Expected no error for shallow pattern with no matches, got %v", err) + } +} + +// TestGetMatchesForPattern_RecursiveMatch tests the recursive branch by overriding glob matching. +func TestGetMatchesForPattern_RecursiveMatch(t *testing.T) { + oldFn := getGlobMatchesForTest + defer func() { getGlobMatchesForTest = oldFn }() + getGlobMatchesForTest = func(pattern string) ([]string, error) { + normalized := filepath.ToSlash(pattern) + if strings.Contains(normalized, "/**") { + return []string{"match.txt"}, nil + } + return []string{}, nil + } + srcDir, err := os.MkdirTemp("", "recursive-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + dir := filepath.Join(srcDir, "dir") + if err := os.Mkdir(dir, 0o755); err != nil { + t.Fatalf("Failed to create directory: %v", err) + } + child := filepath.Join(dir, "child") + if err := os.Mkdir(child, 0o755); err != nil { + t.Fatalf("Failed to create child directory: %v", err) + } + if err := os.WriteFile(filepath.Join(child, "file.txt"), []byte("content"), 0o600); err != nil { + t.Fatalf("Failed to write file: %v", err) + } + pattern := "dir/*/**" + matches, err := getMatchesForPatternForTest(srcDir, pattern) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if len(matches) == 0 { + t.Errorf("Expected matches for recursive branch, got none") + } +} + +// TestIsShallowPattern ensures shallow pattern detection works. +func TestIsShallowPattern(t *testing.T) { + if !isShallowPattern("**/demo-localstack/*") { + t.Errorf("Expected '**/demo-localstack/*' to be shallow") + } + if isShallowPattern("**/demo-library/**") { + t.Errorf("Expected '**/demo-library/**' not to be shallow") + } +} + +// TestCopyDirRecursiveWithPrefix ensures prefix-based copy works. +func TestCopyDirRecursiveWithPrefix(t *testing.T) { + srcDir, err := os.MkdirTemp("", "prefix-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + dstDir, err := os.MkdirTemp("", "prefix-dst") + if err != nil { + t.Fatalf("Failed to create dst dir: %v", err) + } + defer os.RemoveAll(dstDir) + filePath := filepath.Join(srcDir, "test.txt") + if err := os.WriteFile(filePath, []byte("content"), 0o600); err != nil { + t.Fatalf("Failed to write file: %v", err) + } + ctx := &PrefixCopyContext{ + SrcDir: srcDir, + DstDir: dstDir, + GlobalBase: srcDir, + Prefix: "prefix", + Excluded: []string{}, + } + if err := copyDirRecursiveWithPrefix(ctx); err != nil { + t.Fatalf("copyDirRecursiveWithPrefix failed: %v", err) + } + if _, err := os.Stat(filepath.Join(dstDir, "test.txt")); os.IsNotExist(err) { + t.Errorf("Expected file to exist in destination") + } +} + +// TestProcessIncludedPattern ensures that matching files are copied. +func TestProcessIncludedPattern(t *testing.T) { + srcDir, err := os.MkdirTemp("", "included-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + dstDir, err := os.MkdirTemp("", "included-dst") + if err != nil { + t.Fatalf("Failed to create dst dir: %v", err) + } + defer os.RemoveAll(dstDir) + fileMatch := filepath.Join(srcDir, "match.md") + if err := os.WriteFile(fileMatch, []byte("mdcontent"), 0o600); err != nil { + t.Fatalf("Failed to write matching file: %v", err) + } + fileNoMatch := filepath.Join(srcDir, "no_match.txt") + if err := os.WriteFile(fileNoMatch, []byte("txtcontent"), 0o600); err != nil { + t.Fatalf("Failed to write non-matching file: %v", err) + } + pattern := "**/*.md" + if err := processIncludedPattern(srcDir, dstDir, pattern, []string{}); err != nil { + t.Fatalf("processIncludedPattern failed: %v", err) + } + if _, err := os.Stat(filepath.Join(dstDir, "match.md")); os.IsNotExist(err) { + t.Errorf("Expected match.md to exist") + } + if _, err := os.Stat(filepath.Join(dstDir, "no_match.txt")); err == nil { + t.Errorf("Expected no_match.txt not to exist") + } +} + +// TestProcessIncludedPattern_Invalid ensures that an invalid pattern does not cause fatal errors. +func TestProcessIncludedPattern_Invalid(t *testing.T) { + srcDir, err := os.MkdirTemp("", "invalid-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + dstDir, err := os.MkdirTemp("", "invalid-dst") + if err != nil { + t.Fatalf("Failed to create dst dir: %v", err) + } + defer os.RemoveAll(dstDir) + if err := processIncludedPattern(srcDir, dstDir, "[", []string{}); err != nil { + t.Fatalf("Expected processIncludedPattern to handle invalid pattern gracefully, got: %v", err) + } +} + +// TestProcessMatch_ShallowDirectory ensures directories are not copied when shallow is true. +func TestProcessMatch_ShallowDirectory(t *testing.T) { + srcDir, err := os.MkdirTemp("", "pm-shallow-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + dstDir, err := os.MkdirTemp("", "pm-shallow-dst") + if err != nil { + t.Fatalf("Failed to create dst dir: %v", err) + } + defer os.RemoveAll(dstDir) + dirPath := filepath.Join(srcDir, "dir") + if err := os.Mkdir(dirPath, 0o755); err != nil { + t.Fatalf("Failed to create directory: %v", err) + } + if err := processMatch(srcDir, dstDir, dirPath, true, []string{}); err != nil { + t.Errorf("Expected nil error for shallow directory, got %v", err) + } + if _, err := os.Stat(filepath.Join(dstDir, "dir")); err == nil { + t.Errorf("Expected directory not to be copied when shallow is true") + } +} + +// TestProcessMatch_Directory ensures directories are copied when shallow is false. +func TestProcessMatch_Directory(t *testing.T) { + srcDir, err := os.MkdirTemp("", "pm-dir-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + dstDir, err := os.MkdirTemp("", "pm-dir-dst") + if err != nil { + t.Fatalf("Failed to create dst dir: %v", err) + } + defer os.RemoveAll(dstDir) + dirPath := filepath.Join(srcDir, "dir") + if err := os.Mkdir(dirPath, 0o755); err != nil { + t.Fatalf("Failed to create directory: %v", err) + } + fileInside := filepath.Join(dirPath, "inside.txt") + if err := os.WriteFile(fileInside, []byte("data"), 0o600); err != nil { + t.Fatalf("Failed to write file: %v", err) + } + if err := processMatch(srcDir, dstDir, dirPath, false, []string{}); err != nil { + t.Errorf("Expected nil error for directory copy, got %v", err) + } + if _, err := os.Stat(filepath.Join(dstDir, "dir", "inside.txt")); os.IsNotExist(err) { + t.Errorf("Expected file inside directory to be copied") + } +} + +// TestProcessMatch_ErrorStat ensures processMatch returns an error when os.Stat fails. +func TestProcessMatch_ErrorStat(t *testing.T) { + err := processMatch(os.TempDir(), os.TempDir(), "/nonexistentfile.txt", false, []string{}) + if err == nil || !strings.Contains(err.Error(), "stating file") { + t.Errorf("Expected error for non-existent file in processMatch, got %v", err) + } +} + +// TestCopyDirRecursive_ReadDirError checks that copyDirRecursive fails if os.ReadDir fails. +func TestCopyDirRecursive_ReadDirError(t *testing.T) { + ctx := &CopyContext{ + SrcDir: "/nonexistent_directory", + DstDir: os.TempDir(), + BaseDir: "/nonexistent_directory", + Excluded: []string{}, + Included: []string{}, + } + err := copyDirRecursive(ctx) + if err == nil || !strings.Contains(err.Error(), "reading directory") { + t.Errorf("Expected error for non-existent src dir, got %v", err) + } +} + +// TestCopyToTargetWithPatterns checks that included/excluded patterns work. +func TestCopyToTargetWithPatterns(t *testing.T) { + srcDir, err := os.MkdirTemp("", "copyto-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + dstDir, err := os.MkdirTemp("", "copyto-dst") + if err != nil { + t.Fatalf("Failed to create dst dir: %v", err) + } + defer os.RemoveAll(dstDir) + subDir := filepath.Join(srcDir, "sub") + if err := os.Mkdir(subDir, 0o755); err != nil { + t.Fatalf("Failed to create sub dir: %v", err) + } + fileKeep := filepath.Join(subDir, "keep.test") + if err := os.WriteFile(fileKeep, []byte("keep"), 0o600); err != nil { + t.Fatalf("Failed to write keep file: %v", err) + } + fileSkip := filepath.Join(subDir, "skip.test") + if err := os.WriteFile(fileSkip, []byte("skip"), 0o600); err != nil { + t.Fatalf("Failed to write skip file: %v", err) + } + dummy := &schema.AtmosVendorSource{ + IncludedPaths: []string{"**/*.test"}, + ExcludedPaths: []string{"**/skip.test"}, + } + if err := copyToTargetWithPatterns(srcDir, dstDir, dummy, false); err != nil { + t.Fatalf("copyToTargetWithPatterns failed: %v", err) + } + if _, err := os.Stat(filepath.Join(dstDir, "sub", "keep.test")); os.IsNotExist(err) { + t.Errorf("Expected keep.test to exist") + } + if _, err := os.Stat(filepath.Join(dstDir, "sub", "skip.test")); err == nil { + t.Errorf("Expected skip.test not to exist") + } +} + +// TestCopyToTargetWithPatterns_NoPatterns tests the branch using cp.Copy. +func TestCopyToTargetWithPatterns_NoPatterns(t *testing.T) { + srcDir, err := os.MkdirTemp("", "nopattern-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + dstDir, err := os.MkdirTemp("", "nopattern-dst") + if err != nil { + t.Fatalf("Failed to create dst dir: %v", err) + } + defer os.RemoveAll(dstDir) + filePath := filepath.Join(srcDir, "file.txt") + if err := os.WriteFile(filePath, []byte("content"), 0o600); err != nil { + t.Fatalf("Failed to write file: %v", err) + } + dummy := &schema.AtmosVendorSource{ + IncludedPaths: []string{}, + ExcludedPaths: []string{}, + } + if err := copyToTargetWithPatterns(srcDir, dstDir, dummy, false); err != nil { + t.Fatalf("copyToTargetWithPatterns failed: %v", err) + } + if _, err := os.Stat(filepath.Join(dstDir, "file.txt")); os.IsNotExist(err) { + t.Errorf("Expected file.txt to exist in destination") + } +} + +// TestCopyToTargetWithPatterns_LocalFileBranch tests the sourceIsLocalFile branch. +func TestCopyToTargetWithPatterns_LocalFileBranch(t *testing.T) { + srcDir, err := os.MkdirTemp("", "local-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + dstDir, err := os.MkdirTemp("", "local-dst") + if err != nil { + t.Fatalf("Failed to create dst dir: %v", err) + } + defer os.RemoveAll(dstDir) + filePath := filepath.Join(srcDir, "file.txt") + if err := os.WriteFile(filePath, []byte("data"), 0o600); err != nil { + t.Fatalf("Failed to write file: %v", err) + } + + // Pass a target path with a file extension to trigger file-to-file copy. + targetFile := filepath.Join(dstDir, "file.txt") + + dummy := &schema.AtmosVendorSource{ + IncludedPaths: []string{"**/*.txt"}, + ExcludedPaths: []string{}, + } + if err := copyToTargetWithPatterns(srcDir, targetFile, dummy, true); err != nil { + t.Fatalf("copyToTargetWithPatterns failed: %v", err) + } + if _, err := os.Stat(targetFile); os.IsNotExist(err) { + t.Errorf("Expected %q to exist in destination", targetFile) + } +} + +// TestProcessDirEntry_InfoError tests error handling in processDirEntry when Info() fails. +func TestProcessDirEntry_InfoError(t *testing.T) { + ctx := &CopyContext{ + SrcDir: "/dummy", + DstDir: "/dummy", + BaseDir: "/dummy", + Excluded: []string{}, + Included: []string{}, + } + err := processDirEntry(fakeDirEntry{name: "error.txt", err: errForcedInfoError}, ctx) + if err == nil || !strings.Contains(err.Error(), "getting info") { + t.Errorf("Expected error for Info() failure, got %v", err) + } +} + +// TestCopyFile_FailCreateDir simulates failure when creating the destination directory. +func TestCopyFile_FailCreateDir(t *testing.T) { + srcDir, err := os.MkdirTemp("", "copyfile-src") + if err != nil { + t.Fatalf("Failed to create source dir: %v", err) + } + defer os.RemoveAll(srcDir) + srcFile := filepath.Join(srcDir, "test.txt") + content := "copyFileTest" + if err := os.WriteFile(srcFile, []byte(content), 0o600); err != nil { + t.Fatalf("Failed to write source file: %v", err) + } + tmpFile, err := os.CreateTemp("", "non-dir") + if err != nil { + t.Fatalf("Failed to create temporary file: %v", err) + } + tmpFile.Close() + defer os.Remove(tmpFile.Name()) + dstFile := filepath.Join(tmpFile.Name(), "test.txt") + err = copyFile(srcFile, dstFile) + if err == nil || !strings.Contains(err.Error(), "creating destination directory") { + t.Errorf("Expected error creating destination directory, got %v", err) + } +} + +// TestCopyFile_FailChmod simulates failure when setting file permissions. +// If the patch doesn't take effect, the test will be skipped. +func TestCopyFile_FailChmod(t *testing.T) { + patches := gomonkey.ApplyFunc(os.Chmod, func(name string, mode os.FileMode) error { + return errSimulatedChmodFailure + }) + defer patches.Reset() + + srcDir, err := os.MkdirTemp("", "copyfile-src") + if err != nil { + t.Fatalf("Failed to create source dir: %v", err) + } + defer os.RemoveAll(srcDir) + dstDir, err := os.MkdirTemp("", "copyfile-dst") + if err != nil { + t.Fatalf("Failed to create destination dir: %v", err) + } + defer os.RemoveAll(dstDir) + srcFile := filepath.Join(srcDir, "test.txt") + content := "copyFileTest" + if err := os.WriteFile(srcFile, []byte(content), 0o600); err != nil { + t.Fatalf("Failed to write source file: %v", err) + } + dstFile := filepath.Join(dstDir, "test.txt") + err = copyFile(srcFile, dstFile) + if err == nil { + t.Skip("os.Chmod patch not effective on this platform") + } + if !strings.Contains(err.Error(), "setting permissions") { + t.Errorf("Expected chmod error, got %v", err) + } +} + +// TestGetMatchesForPattern_GlobError forces u.GetGlobMatches to return an error. +func TestGetMatchesForPattern_GlobError(t *testing.T) { + patches := gomonkey.ApplyFunc(u.GetGlobMatches, func(pattern string) ([]string, error) { + return nil, errSimulatedGlobError + }) + defer patches.Reset() + + srcDir := "/dummy/src" + pattern := "*.txt" + _, err := getMatchesForPattern(srcDir, pattern) + if err == nil || !strings.Contains(err.Error(), "simulated glob error") { + t.Errorf("Expected simulated glob error, got %v", err) + } +} + +// TestInclusionExclusion_TrailingSlash tests the trailing-slash branch in shouldExcludePath for directories. +func TestInclusionExclusion_TrailingSlash(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "testdir") + if err != nil { + t.Fatalf("Failed to create temporary directory: %v", err) + } + defer os.RemoveAll(tmpDir) + + info, err := os.Stat(tmpDir) + if err != nil { + t.Fatalf("Failed to stat temporary directory: %v", err) + } + + relPath := filepath.Base(tmpDir) + + // Test that the directory is excluded when the exclusion pattern expects a trailing slash. + if !shouldExcludePath(info, relPath, []string{relPath + "/"}) { + t.Errorf("Expected directory %q to be excluded by pattern %q", relPath, relPath+"/") + } + + // Test that the directory is not excluded when the pattern does not match. + if shouldExcludePath(info, relPath, []string{relPath + "x/"}) { + t.Errorf("Did not expect directory %q to be excluded by pattern %q", relPath, relPath+"x/") + } +} + +// TestProcessPrefixEntry_InfoError simulates an error when calling Info() in processPrefixEntry. +func TestProcessPrefixEntry_InfoError(t *testing.T) { + ctx := &PrefixCopyContext{ + SrcDir: "dummySrc", + DstDir: "dummyDst", + GlobalBase: "dummyGlobal", + Prefix: "dummyPrefix", + Excluded: []string{}, + } + fakeEntry := fakeDirEntry{ + name: "error.txt", + err: errForcedInfoError, + } + err := processPrefixEntry(fakeEntry, ctx) + if err == nil || !strings.Contains(err.Error(), "getting info") { + t.Errorf("Expected error getting info, got %v", err) + } +} + +// fakeFileInfo is a minimal implementation of os.FileInfo for testing. +type fakeFileInfo struct { + name string + size int64 + mode os.FileMode + isDir bool +} + +func (fi fakeFileInfo) Name() string { return fi.name } +func (fi fakeFileInfo) Size() int64 { return fi.size } +func (fi fakeFileInfo) Mode() os.FileMode { return fi.mode } +func (fi fakeFileInfo) ModTime() time.Time { return time.Time{} } +func (fi fakeFileInfo) IsDir() bool { return fi.isDir } +func (fi fakeFileInfo) Sys() any { return nil } + +// fakeDirEntryWithInfo implements os.DirEntry using fakeFileInfo. +type fakeDirEntryWithInfo struct { + name string + info os.FileInfo +} + +func (fde fakeDirEntryWithInfo) Name() string { return fde.name } +func (fde fakeDirEntryWithInfo) IsDir() bool { return fde.info.IsDir() } +func (fde fakeDirEntryWithInfo) Type() os.FileMode { return fde.info.Mode() } +func (fde fakeDirEntryWithInfo) Info() (os.FileInfo, error) { return fde.info, nil } + +// TestProcessPrefixEntry_FailMkdir simulates an error when creating a directory in processPrefixEntry. +func TestProcessPrefixEntry_FailMkdir(t *testing.T) { + srcDir, err := os.MkdirTemp("", "prefix-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + dstDir, err := os.MkdirTemp("", "prefix-dst") + if err != nil { + t.Fatalf("Failed to create dst dir: %v", err) + } + defer os.RemoveAll(dstDir) + + // Create a fake directory entry. + fi := fakeFileInfo{ + name: "testDir", + mode: 0o755, + isDir: true, + } + fakeEntry := fakeDirEntryWithInfo{ + name: "testDir", + info: fi, + } + + ctx := &PrefixCopyContext{ + SrcDir: srcDir, + DstDir: dstDir, + GlobalBase: srcDir, + Prefix: "prefix", + Excluded: []string{}, + } + + patches := gomonkey.ApplyFunc(os.MkdirAll, func(path string, perm os.FileMode) error { + return errSimulatedMkdirAllError + }) + defer patches.Reset() + + err = processPrefixEntry(fakeEntry, ctx) + if err == nil || !strings.Contains(err.Error(), "creating directory") { + t.Errorf("Expected error creating directory, got %v", err) + } +} + +// TestCopyToTargetWithPatterns_UseCpCopy ensures that when no inclusion/exclusion patterns are defined, the cp.Copy branch is used. +func TestCopyToTargetWithPatterns_UseCpCopy(t *testing.T) { + srcDir, err := os.MkdirTemp("", "nopattern-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + dstDir, err := os.MkdirTemp("", "nopattern-dst") + if err != nil { + t.Fatalf("Failed to create dst dir: %v", err) + } + defer os.RemoveAll(dstDir) + + // Create a test file in the source directory. + filePath := filepath.Join(srcDir, "file.txt") + if err := os.WriteFile(filePath, []byte("content"), 0o600); err != nil { + t.Fatalf("Failed to write file: %v", err) + } + + // Patch the cp.Copy function to verify that it is called. + called := false + patch := gomonkey.ApplyFunc(cp.Copy, func(src, dst string) error { + called = true + // For testing purposes, simulate a copy by using our copyFile function. + return copyFile(filepath.Join(src, "file.txt"), filepath.Join(dst, "file.txt")) + }) + defer patch.Reset() + + dummy := &schema.AtmosVendorSource{ + IncludedPaths: []string{}, + ExcludedPaths: []string{}, + } + if err := copyToTargetWithPatterns(srcDir, dstDir, dummy, false); err != nil { + t.Fatalf("copyToTargetWithPatterns failed: %v", err) + } + if !called { + t.Errorf("Expected cp.Copy to be called, but it was not") + } + // Verify that the file was "copied" to the destination. + if _, err := os.Stat(filepath.Join(dstDir, "file.txt")); os.IsNotExist(err) { + t.Errorf("Expected file.txt to exist in destination") + } +} + +// TestGetMatchesForPattern_ShallowNoMatches tests a shallow pattern (ending with "/*" but not "/**") +// when no matches are found, expecting an empty result. +func TestGetMatchesForPattern_ShallowNoMatches(t *testing.T) { + patches := gomonkey.ApplyFunc(u.GetGlobMatches, func(pattern string) ([]string, error) { + return []string{}, nil + }) + defer patches.Reset() + + srcDir := "/dummy/src" + pattern := "dummy/*" // Shallow pattern without recursive "**" + matches, err := getMatchesForPattern(srcDir, pattern) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if len(matches) != 0 { + t.Errorf("Expected no matches for shallow pattern, got %v", matches) + } +} + +// TestProcessMatch_RelPathError simulates an error in computing the relative path in processMatch. +func TestProcessMatch_RelPathError(t *testing.T) { + srcDir := "/dummy/src" + dstPath := "/dummy/dst" + + // Create a temporary file to act as the target file. + tmpFile, err := os.CreateTemp("", "relerr") + if err != nil { + t.Fatalf("Failed to create temporary file: %v", err) + } + tmpFile.Close() + defer os.Remove(tmpFile.Name()) + filePath := tmpFile.Name() + + patches := gomonkey.ApplyFunc(filepath.Rel, func(basepath, targpath string) (string, error) { + return "", errSimulatedRelPathError + }) + defer patches.Reset() + + err = processMatch(srcDir, dstPath, filePath, false, []string{}) + if err == nil || !strings.Contains(err.Error(), "computing relative path") { + t.Errorf("Expected relative path error, got %v", err) + } +} diff --git a/internal/exec/error.go b/internal/exec/error.go new file mode 100644 index 0000000000..291cecd46f --- /dev/null +++ b/internal/exec/error.go @@ -0,0 +1,22 @@ +package exec + +import ( + "errors" +) + +var ( + ErrDownloadPackage = errors.New("failed to download package") + ErrProcessOCIImage = errors.New("failed to process OCI image") + ErrCopyPackage = errors.New("failed to copy package") + ErrCreateTempDir = errors.New("failed to create temp directory") + ErrUnknownPackageType = errors.New("unknown package type") + ErrLocalMixinURICannotBeEmpty = errors.New("local mixin URI cannot be empty") + ErrLocalMixinInstallationNotImplemented = errors.New("local mixin installation not implemented") + ErrFailedToInitializeTUIModel = errors.New("failed to initialize TUI model: verify terminal capabilities and permissions") + ErrSetTempDirPermissions = errors.New("failed to set temp directory permissions") + ErrCopyPackageToTarget = errors.New("failed to copy package to target") + ErrNoValidInstallerPackage = errors.New("no valid installer package provided") + ErrFailedToInitializeTUIModelWithDetails = errors.New("failed to initialize TUI model: verify terminal capabilities and permissions") + ErrValidPackage = errors.New("no valid installer package provided for") + ErrTUIModel = errors.New("failed to initialize TUI model") +) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index 29e9a415e8..081728cbeb 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -11,6 +11,7 @@ import ( "strings" "time" + log "github.com/charmbracelet/log" "github.com/google/uuid" "github.com/hashicorp/go-getter" @@ -18,6 +19,8 @@ import ( u "github.com/cloudposse/atmos/pkg/utils" ) +const keyURL = "url" + // ValidateURI validates URIs func ValidateURI(uri string) error { if uri == "" { @@ -61,15 +64,14 @@ func IsValidScheme(scheme string) bool { return validSchemes[scheme] } -// CustomGitHubDetector intercepts GitHub URLs and transforms them -// into something like git::https://@github.com/... so we can -// do a git-based clone with a token. -type CustomGitHubDetector struct { +// CustomGitDetector intercepts GitHub URLs and transforms them into something like git::https://@github.com/ so we can do a git-based clone with a token. +type CustomGitDetector struct { AtmosConfig schema.AtmosConfiguration + source string } // Detect implements the getter.Detector interface for go-getter v1. -func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) { +func (d *CustomGitDetector) Detect(src, _ string) (string, bool, error) { if len(src) == 0 { return "", false, nil } @@ -128,23 +130,52 @@ func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) { } } + // Adjust subdirectory if needed. + d.adjustSubdir(parsedURL, d.source) + + // Set "depth=1" for a shallow clone if not specified. + q := parsedURL.Query() + if _, exists := q["depth"]; !exists { + q.Set("depth", "1") + } + parsedURL.RawQuery = q.Encode() + finalURL := "git::" + parsedURL.String() + maskedFinal, err := u.MaskBasicAuth(strings.TrimPrefix(finalURL, "git::")) + if err != nil { + log.Debug("Masking failed", "error", err) + } else { + log.Debug("Final URL (masked)", "url", "git::"+maskedFinal) + } return finalURL, true, nil } +// adjustSubdir appends "//." to the path if no subdirectory is specified. +func (d *CustomGitDetector) adjustSubdir(parsedURL *url.URL, source string) { + normalizedSource := filepath.ToSlash(source) + if normalizedSource != "" && !strings.Contains(normalizedSource, "//") { + parts := strings.SplitN(parsedURL.Path, "/", 4) + if strings.HasSuffix(parsedURL.Path, ".git") || len(parts) == 3 { + maskedSrc, _ := u.MaskBasicAuth(source) + log.Debug("Detected top-level repo with no subdir: appending '//.'", keyURL, maskedSrc) + parsedURL.Path += "//." + } + } +} + // RegisterCustomDetectors prepends the custom detector so it runs before // the built-in ones. Any code that calls go-getter should invoke this. -func RegisterCustomDetectors(atmosConfig schema.AtmosConfiguration) { +func RegisterCustomDetectors(atmosConfig schema.AtmosConfiguration, source string) { getter.Detectors = append( []getter.Detector{ - &CustomGitHubDetector{AtmosConfig: atmosConfig}, + &CustomGitDetector{AtmosConfig: atmosConfig, source: source}, }, getter.Detectors..., ) } -// GoGetterGet downloads packages (files and folders) from different sources using `go-getter` and saves them into the destination +// GoGetterGet downloads packages (files and folders) from different sources using `go-getter` and saves them into the destination. func GoGetterGet( atmosConfig schema.AtmosConfiguration, src string, @@ -155,8 +186,11 @@ func GoGetterGet( ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - // Register custom detectors - RegisterCustomDetectors(atmosConfig) + // Register custom detectors, passing the original `src` to the CustomGitDetector. + // go-getter typically strips subdirectories before calling the detector, so the + // unaltered source is needed to identify whether a top-level repository or a + // subdirectory was specified (e.g., for appending "//." only when no subdir is present). + RegisterCustomDetectors(atmosConfig, src) client := &getter.Client{ Ctx: ctx, @@ -164,8 +198,17 @@ func GoGetterGet( // Destination where the files will be stored. This will create the directory if it doesn't exist Dst: dest, Mode: clientMode, + Getters: map[string]getter.Getter{ + // Overriding 'git' + "git": &CustomGitGetter{}, + "file": &getter.FileGetter{}, + "hg": &getter.HgGetter{}, + "http": &getter.HttpGetter{}, + "https": &getter.HttpGetter{}, + // "s3": &getter.S3Getter{}, // add as needed + // "gcs": &getter.GCSGetter{}, + }, } - if err := client.Get(); err != nil { return err } @@ -173,12 +216,43 @@ func GoGetterGet( return nil } -// DownloadDetectFormatAndParseFile downloads a remote file, detects the format of the file (JSON, YAML, HCL) and parses the file into a Go type -func DownloadDetectFormatAndParseFile(atmosConfig schema.AtmosConfiguration, file string) (any, error) { +// CustomGitGetter is a custom getter for git (git::) that removes symlinks. +type CustomGitGetter struct { + getter.GitGetter +} + +// Get implements the custom getter logic removing symlinks. +func (c *CustomGitGetter) Get(dst string, url *url.URL) error { + // Normal clone + if err := c.GitGetter.Get(dst, url); err != nil { + return err + } + // Remove symlinks + return removeSymlinks(dst) +} + +// removeSymlinks walks the directory and removes any symlinks +// it encounters. +func removeSymlinks(root string) error { + return filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.Mode()&os.ModeSymlink != 0 { + log.Debug("Removing symlink", "path", path) + // Symlinks are removed for the entire repo, regardless if there are any subfolders specified + return os.Remove(path) + } + return nil + }) +} + +// DownloadDetectFormatAndParseFile downloads a remote file, detects the format of the file (JSON, YAML, HCL) and parses the file into a Go type. +func DownloadDetectFormatAndParseFile(atmosConfig *schema.AtmosConfiguration, file string) (any, error) { tempDir := os.TempDir() f := filepath.Join(tempDir, uuid.New().String()) - if err := GoGetterGet(atmosConfig, file, f, getter.ClientModeFile, time.Second*30); err != nil { + if err := GoGetterGet(*atmosConfig, file, f, getter.ClientModeFile, 30*time.Second); err != nil { return nil, fmt.Errorf("failed to download the file '%s': %w", file, err) } @@ -202,41 +276,40 @@ scp, sftp Shortcuts like github.com, bitbucket.org - File-related Schemes: -file - Local filesystem paths -dir - Local directories -tar - Tar files, potentially compressed (tar.gz, tar.bz2, etc.) -zip - Zip files + file - Local filesystem paths + dir - Local directories + tar - Tar files, potentially compressed (tar.gz, tar.bz2, etc.) + zip - Zip files - HTTP/HTTPS: -http - HTTP URLs -https - HTTPS URLs + http - HTTP URLs + https - HTTPS URLs - Git: -git - Git repositories, which can be accessed via HTTPS or SSH + git - Git repositories, which can be accessed via HTTPS or SSH - Mercurial: -hg - Mercurial repositories, accessed via HTTP/S or SSH + hg - Mercurial repositories, accessed via HTTP/S or SSH - Amazon S3: -s3 - Amazon S3 bucket URLs + s3 - Amazon S3 bucket URLs - Google Cloud Storage: -gcs - Google Cloud Storage URLs + gcs - Google Cloud Storage URLs - OCI: -oci - Open Container Initiative (OCI) images + oci - Open Container Initiative (OCI) images - Other Protocols: -scp - Secure Copy Protocol for SSH-based transfers -sftp - SSH File Transfer Protocol + scp - Secure Copy Protocol for SSH-based transfers + sftp - SSH File Transfer Protocol - GitHub/Bitbucket/Other Shortcuts: -github.com - Direct GitHub repository shortcuts -bitbucket.org - Direct Bitbucket repository shortcuts + github.com - Direct GitHub repository shortcuts + bitbucket.org - Direct Bitbucket repository shortcuts - Composite Schemes: -go-getter allows for composite schemes, where multiple operations can be combined. For example: -git::https://github.com/user/repo - Forces the use of git over an HTTPS URL. -tar::http://example.com/archive.tar.gz - Treats the HTTP resource as a tarball. - + go-getter allows for composite schemes, where multiple operations can be combined. For example: + git::https://github.com/user/repo - Forces the use of git over an HTTPS URL. + tar::http://example.com/archive.tar.gz - Treats the HTTP resource as a tarball. */ diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go index cc292d58af..d19b68c71e 100644 --- a/internal/exec/vendor_model.go +++ b/internal/exec/vendor_model.go @@ -1,19 +1,17 @@ package exec import ( - "errors" "fmt" "os" "path/filepath" "strings" "time" - log "github.com/charmbracelet/log" - "github.com/charmbracelet/bubbles/progress" "github.com/charmbracelet/bubbles/spinner" tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/lipgloss" + log "github.com/charmbracelet/log" "github.com/hashicorp/go-getter" cp "github.com/otiai10/copy" @@ -34,14 +32,11 @@ const ( ) var ( - ErrValidPackage = errors.New("no valid installer package provided for") - ErrTUIModel = errors.New("failed to initialize TUI model") - ErrUnknownPackageType = errors.New("unknown package type") - currentPkgNameStyle = theme.Styles.PackageName - doneStyle = lipgloss.NewStyle().Margin(1, 2) - checkMark = theme.Styles.Checkmark - xMark = theme.Styles.XMark - grayColor = theme.Styles.GrayText + currentPkgNameStyle = theme.Styles.PackageName + doneStyle = lipgloss.NewStyle().Margin(1, 2) + checkMark = theme.Styles.Checkmark + xMark = theme.Styles.XMark + grayColor = theme.Styles.GrayText ) type installedPkgMsg struct { @@ -347,7 +342,7 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig *schema.Atmo if err := p.installer(&tempDir, atmosConfig); err != nil { return newInstallError(err, p.name) } - if err := copyToTarget(tempDir, p.targetPath, &p.atmosVendorSource, p.sourceIsLocalFile, p.uri); err != nil { + if err := copyToTargetWithPatterns(tempDir, p.targetPath, &p.atmosVendorSource, p.sourceIsLocalFile); err != nil { return newInstallError(fmt.Errorf("failed to copy package: %w", err), p.name) } return installedPkgMsg{ diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index 96b84ba94c..9254b0627a 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -19,6 +19,9 @@ import ( u "github.com/cloudposse/atmos/pkg/utils" ) +// Dedicated logger for stderr to keep stdout clean of detailed messaging, e.g. for files vendoring. +var StderrLogger = log.New(os.Stderr) + var ( ErrVendorComponents = errors.New("failed to vendor components") ErrSourceMissing = errors.New("'source' must be specified in 'sources' in the vendor config file") diff --git a/internal/exec/yaml_func_include.go b/internal/exec/yaml_func_include.go index 4d75f3db06..5cc18a46ca 100644 --- a/internal/exec/yaml_func_include.go +++ b/internal/exec/yaml_func_include.go @@ -47,7 +47,7 @@ func processTagInclude( if fileType == u.AtmosYamlFuncIncludeLocalFile { res, err = u.DetectFormatAndParseFile(f) } else if fileType == u.AtmosYamlFuncIncludeGoGetter { - res, err = DownloadDetectFormatAndParseFile(atmosConfig, f) + res, err = DownloadDetectFormatAndParseFile(&atmosConfig, f) } if err != nil { diff --git a/pkg/utils/url_utils.go b/pkg/utils/url_utils.go new file mode 100644 index 0000000000..e176c377fe --- /dev/null +++ b/pkg/utils/url_utils.go @@ -0,0 +1,20 @@ +package utils + +import ( + "fmt" + "net/url" +) + +// MaskBasicAuth replaces the username and password in a URL with "xxx" if present. +func MaskBasicAuth(rawURL string) (string, error) { + parsedURL, err := url.Parse(rawURL) + if err != nil { + return "", fmt.Errorf("failed to parse URL: %w", err) + } + + if parsedURL.User != nil { + parsedURL.User = url.UserPassword("xxx", "xxx") + } + + return parsedURL.String(), nil +} diff --git a/tests/cli_test.go b/tests/cli_test.go index 8679eff212..230d1e7b9b 100644 --- a/tests/cli_test.go +++ b/tests/cli_test.go @@ -46,13 +46,14 @@ var ( var logger *log.Logger type Expectation struct { - Stdout []MatchPattern `yaml:"stdout"` // Expected stdout output - Stderr []MatchPattern `yaml:"stderr"` // Expected stderr output - ExitCode int `yaml:"exit_code"` // Expected exit code - FileExists []string `yaml:"file_exists"` // Files to validate - FileContains map[string][]MatchPattern `yaml:"file_contains"` // File contents to validate (file to patterns map) - Diff []string `yaml:"diff"` // Acceptable differences in snapshot - Timeout string `yaml:"timeout"` // Maximum execution time as a string, e.g., "1s", "1m", "1h", or a number (seconds) + Stdout []MatchPattern `yaml:"stdout"` // Expected stdout output + Stderr []MatchPattern `yaml:"stderr"` // Expected stderr output + ExitCode int `yaml:"exit_code"` // Expected exit code + FileExists []string `yaml:"file_exists"` // Files to validate + FileNotExists []string `yaml:"file_not_exists"` // Files that should not exist + FileContains map[string][]MatchPattern `yaml:"file_contains"` // File contents to validate (file to patterns map) + Diff []string `yaml:"diff"` // Acceptable differences in snapshot + Timeout string `yaml:"timeout"` // Maximum execution time as a string, e.g., "1s", "1m", "1h", or a number (seconds) } type TestCase struct { Name string `yaml:"name"` // Name of the test @@ -687,6 +688,11 @@ func runCLICommandTest(t *testing.T, tc TestCase) { t.Errorf("Description: %s", tc.Description) } + // Validate file not existence + if !verifyFileNotExists(t, tc.Expect.FileNotExists) { + t.Errorf("Description: %s", tc.Description) + } + // Validate file contents if !verifyFileContains(t, tc.Expect.FileContains) { t.Errorf("Description: %s", tc.Description) @@ -795,6 +801,20 @@ func verifyFileExists(t *testing.T, files []string) bool { return success } +func verifyFileNotExists(t *testing.T, files []string) bool { + success := true + for _, file := range files { + if _, err := os.Stat(file); err == nil { + t.Errorf("Reason: File %q exists but it should not.", file) + success = false + } else if !errors.Is(err, os.ErrNotExist) { + t.Errorf("Reason: Unexpected error checking file %q: %v", file, err) + success = false + } + } + return success +} + func verifyFileContains(t *testing.T, filePatterns map[string][]MatchPattern) bool { success := true for file, patterns := range filePatterns { diff --git a/tests/fixtures/scenarios/vendor-globs/atmos.yaml b/tests/fixtures/scenarios/vendor-globs/atmos.yaml new file mode 100644 index 0000000000..0f0506e81a --- /dev/null +++ b/tests/fixtures/scenarios/vendor-globs/atmos.yaml @@ -0,0 +1,40 @@ +base_path: "./" + +components: + terraform: + base_path: "components/terraform" + apply_auto_approve: false + deploy_run_init: true + init_run_reconfigure: true + auto_generate_backend_file: false + +stacks: + base_path: "stacks" + included_paths: + - "deploy/**/*" + excluded_paths: + - "**/_defaults.yaml" + name_pattern: "{stage}" + +vendor: + # Single file + base_path: "./vendor.yaml" + + # Directory with multiple files + #base_path: "./vendor" + + # Absolute path + #base_path: "vendor.d/vendor1.yaml" + +logs: + file: "/dev/stderr" + level: Info + +# Custom CLI commands + +# No arguments or flags are required +commands: +- name: "test" + description: "Run all tests" + steps: + - atmos vendor pull --everything diff --git a/tests/fixtures/scenarios/vendor-globs/vendor.yaml b/tests/fixtures/scenarios/vendor-globs/vendor.yaml new file mode 100644 index 0000000000..061c06d19c --- /dev/null +++ b/tests/fixtures/scenarios/vendor-globs/vendor.yaml @@ -0,0 +1,43 @@ +apiVersion: atmos/v1 +kind: AtmosVendorConfig +metadata: + name: demo-vendoring + description: Atmos vendoring manifest for Atmos demo component library +spec: + # Import other vendor manifests, if necessary + imports: [] + + sources: + - component: "test globs" + source: "github.com/cloudposse/atmos.git" + included_paths: + - "**/{demo-library,demo-stacks}/**/*.{tf,md}" + excluded_paths: + - "**/demo-library/**/*.{tfvars,tf}" + targets: + - "components/library/" + tags: + - demo + + - component: "test globs without double stars upfront" + source: "github.com/cloudposse/atmos.git//examples/demo-library?ref={{.Version}}" + included_paths: + - "/weather/*.md" + version: "main" + targets: + - "components/library/" + tags: + - demo + + - component: "test shallow globs and folder exclusion" + source: "github.com/cloudposse/atmos.git" + included_paths: + - "**/demo-localstack/*" + - "**/demo-library/**" + excluded_paths: + - "**/demo-library/**/stargazers/**" + - "**/demo-library/**/*.tf" + targets: + - "components/globs/" + tags: + - demo diff --git a/tests/fixtures/scenarios/vendor/vendor.yaml b/tests/fixtures/scenarios/vendor/vendor.yaml index c4ace43366..ee3020ec65 100644 --- a/tests/fixtures/scenarios/vendor/vendor.yaml +++ b/tests/fixtures/scenarios/vendor/vendor.yaml @@ -51,3 +51,4 @@ spec: - "**/*.tftmpl" - "**/modules/**" excluded_paths: [] + diff --git a/tests/test-cases/demo-globs.yaml b/tests/test-cases/demo-globs.yaml new file mode 100644 index 0000000000..1b29cb0181 --- /dev/null +++ b/tests/test-cases/demo-globs.yaml @@ -0,0 +1,61 @@ +tests: + - name: atmos_vendor_pull_with_globs + enabled: true + description: "Ensure atmos vendor pull command executes without errors and files are present." + workdir: "fixtures/scenarios/vendor-globs" + command: "atmos" + args: + - "vendor" + - "pull" + expect: + file_exists: + - "./components/library/examples/demo-library/github/stargazers/README.md" + - "./components/library/examples/demo-library/ipinfo/README.md" + - "./components/library/examples/demo-library/weather/README.md" + - "./components/library/examples/demo-library/README.md" + - "./components/library/examples/demo-stacks/components/terraform/myapp/main.tf" + - "./components/library/examples/demo-stacks/components/terraform/myapp/outputs.tf" + - "./components/library/examples/demo-stacks/components/terraform/myapp/README.md" + - "./components/library/examples/demo-stacks/components/terraform/myapp/variables.tf" + - "./components/library/examples/demo-stacks/components/terraform/myapp/versions.tf" + - "./components/library/weather/README.md" + - "./components/globs/examples/demo-library/ipinfo/README.md" + - "./components/globs/examples/demo-library/weather/README.md" + - "./components/globs/examples/demo-library/README.md" + - "./components/globs/examples/demo-localstack/.gitignore" + - "./components/globs/examples/demo-localstack/atmos.yaml" + - "./components/globs/examples/demo-localstack/docker-compose.yml" + - "./components/globs/examples/demo-localstack/README.md" + file_not_exists: + - "./components/library/examples/demo-workflows/stacks/catalog/myapp.yaml" + - "./components/library/examples/demo-library/github/stargazers/main.tf" + - "./components/library/examples/demo-library/github/stargazers/outputs.tf" + - "./components/library/examples/demo-library/github/stargazers/providers.tf" + - "./components/library/examples/demo-library/github/stargazers/variables.tf" + - "./components/library/examples/demo-library/github/stargazers/versions.tf" + - "./components/library/examples/demo-library/ipinfo/main.tf" + - "./components/library/examples/demo-library/ipinfo/outputs.tf" + - "./components/library/examples/demo-library/ipinfo/providers.tf" + - "./components/library/examples/demo-library/ipinfo/variables.tf" + - "./components/library/examples/demo-library/ipinfo/versions.tf" + - "./components/library/examples/demo-library/weather/main.tf" + - "./components/library/examples/demo-library/weather/outputs.tf" + - "./components/library/examples/demo-library/weather/providers.tf" + - "./components/library/examples/demo-library/weather/variables.tf" + - "./components/library/examples/demo-library/weather/versions.tf" + - "./components/globs/examples/demo-library/github/stargazers/README.md" + - "./components/globs/examples/demo-library/github/stargazers/main.tf" + - "./components/globs/examples/demo-library/github/stargazers/outputs.tf" + - "./components/globs/examples/demo-library/github/stargazers/providers.tf" + - "./components/globs/examples/demo-library/github/stargazers/variables.tf" + - "./components/globs/examples/demo-library/github/stargazers/versions.tf" + - "./components/globs/examples/demo-library/ipinfo/outputs.tf" + - "./components/globs/examples/demo-library/ipinfo/providers.tf" + - "./components/globs/examples/demo-library/ipinfo/variables.tf" + - "./components/globs/examples/demo-library/ipinfo/versions.tf" + - "./components/globs/examples/demo-library/weather/main.tf" + - "./components/globs/examples/demo-library/weather/outputs.tf" + - "./components/globs/examples/demo-library/weather/providers.tf" + - "./components/globs/examples/demo-library/weather/variables.tf" + - "./components/globs/examples/demo-library/weather/versions.tf" + exit_code: 0 diff --git a/website/docs/core-concepts/vendor/vendor-manifest.mdx b/website/docs/core-concepts/vendor/vendor-manifest.mdx index c41e2fec08..0fe114f2e2 100644 --- a/website/docs/core-concepts/vendor/vendor-manifest.mdx +++ b/website/docs/core-concepts/vendor/vendor-manifest.mdx @@ -217,7 +217,7 @@ The `vendor.yaml` vendoring manifest supports Kubernetes-style YAML config to de
`included_paths` and `excluded_paths`
- `included_paths` and `excluded_paths` support [POSIX-style greedy Globs](https://en.wikipedia.org/wiki/Glob_(programming)) for filenames/paths (double-star/globstar `**` is supported as well). + `included_paths` and `excluded_paths` support [POSIX-style greedy Globs](https://en.wikipedia.org/wiki/Glob_(programming)) for filenames/paths (double-star/globstar `**` is supported as well). For more details, see [Vendoring with Globs](#vendoring-with-globs).
`component`
@@ -497,3 +497,115 @@ To vendor the `vpc` component, execute the following command: atmos vendor pull -c vpc ``` + +## Vendoring with Globs + +In Atmos, **glob patterns** define which files and directories are included or excluded during vendoring. These patterns go beyond simple wildcard characters like `*`—they follow specific rules that dictate how paths are matched. Understanding the difference between **greedy** (`**`) and **non-greedy** (`*`) patterns, along with other advanced glob syntax, ensures precise control over vendoring behavior. + +### Understanding Wildcards, Ranges, and Recursion + +Glob patterns in Atmos provide flexible and powerful matching, that's simpler to understand than regular expressions: + +
+
`*` (single asterisk)
+
Matches any sequence of characters within a single path segment.
+
Example: `vendor/*.yaml` matches `vendor/config.yaml` but not `vendor/subdir/config.yaml`.
+ +
`**` (double asterisk, also known as a "greedy glob")
+
Matches across multiple path segments recursively.
+
Example: `vendor/**/*.yaml` matches `vendor/config.yaml`, `vendor/subdir/config.yaml`, and `vendor/deep/nested/config.yaml`.
+ +
`?` (question mark)
+
Matches exactly one character in a path segment.
+
Example: `file?.txt` matches `file1.txt` and `fileA.txt` but not `file10.txt`.
+ +
`[abc]` (character class)
+
Matches any single character inside the brackets.
+
Example: `file[123].txt` matches `file1.txt`, `file2.txt`, and `file3.txt`, but not `file4.txt` or `file12.txt`.
+ +
`[a-z]` (character range)
+
Matches any single character within the specified range.
+
Example: `file[a-c].txt` matches `filea.txt`, `fileb.txt`, and `filec.txt`.
+ +
`{a,b,c}` (brace expansion)
+
Matches any of the comma-separated patterns.
+
Example: `*.{jpg,png,gif}` matches `image.jpg`, `image.png`, and `image.gif`.
+
+ +This distinction is important when excluding specific directories or files while vendoring. + +#### Example: Excluding a Subdirectory + +Consider the following configuration: + +```yaml +included_paths: + - "**/demo-library/**" +excluded_paths: + - "**/demo-library/**/stargazers/**" +``` + +How it works: +- The `included_paths` rule `**/demo-library/**` ensures all files inside `demo-library` (at any depth) are vendored. +- The `excluded_paths` rule `**/demo-library/**/stargazers/**` prevents any files inside `stargazers` subdirectories from being vendored. + +This means: +- All files within `demo-library` except those inside any `stargazers` subdirectory are vendored. +- Any other files outside `stargazers` are unaffected by this exclusion. + +#### Example: A Non-Recursive Pattern That Doesn't Work + +```yaml +included_paths: + - "**/demo-library/*" +excluded_paths: + - "**/demo-library/**/stargazers/**" +``` + +In this case: +- `**/demo-library/*` only matches immediate children of `demo-library`, not nested files or subdirectories. +- This means `stargazers/` itself could be matched, but its contents might not be explicitly excluded. +- To correctly capture all subdirectories and files while still excluding stargazers, use `**/demo-library/**/*`. + +Using `{...}` for Multiple Extensions or Patterns + +Curly braces `{...}` allow for expanding multiple patterns into separate glob matches. This is useful when selecting multiple file types or directories within a single glob pattern. + +#### Example: Matching Multiple File Extensions + +```yaml +included_paths: + - "**/demo-library/**/*.{tf,md}" +``` + +This is equivalent to writing: + +```yaml +included_paths: + - "**/demo-library/**/*.tf" + - "**/demo-library/**/*.md" +``` + +The `{tf,md}` part expands to both `*.tf` and `*.md`, making the rule more concise. + +#### Example: Excluding Multiple Directories + +```yaml +excluded_paths: + - "**/demo-library/**/{stargazers,archive}/**" +``` + +This excludes both: +- `**/demo-library/**/stargazers/**` +- `**/demo-library/**/archive/**` + +Using `{...}` here prevents the need to write two separate exclusion rules. + +## Key Takeaways + +1. Use `**/` for recursive matching to include everything inside a directory. +2. Use `*` for single-segment matches, which won't include deeper subdirectories. +3. Use `{...}` to match multiple extensions or directories within a single pattern. +4. Exclusion rules must match nested paths explicitly when trying to exclude deep directories. + +By carefully combining `included_paths`, `excluded_paths`, and `{...}` expansion, you can precisely control which files are vendored while ensuring unwanted directories are omitted. \ No newline at end of file