feat: support discovery of symlinked modules

Signed-off-by: Rodrigo Fior Kuntzer <[email protected]>

cli/commands/catalog/module/repo.go

@@ -9,6 +9,8 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/gruntwork-io/terragrunt/util"
+
 	"github.com/gitsight/go-vcsurl"
 	"github.com/gruntwork-io/go-commons/files"
 	"github.com/gruntwork-io/terragrunt/internal/errors"
@@ -82,7 +84,7 @@ func (repo *Repo) FindModules(ctx context.Context) (Modules, error) {
 			continue
 		}
 
-		err := filepath.Walk(modulesPath,
+		err := util.WalkWithSymlinks(modulesPath,
 			func(dir string, remote os.FileInfo, err error) error {
 				if err != nil {
 					return err

config/config.go

@@ -650,7 +650,7 @@ func GetDefaultConfigPath(workingDir string) string {
 func FindConfigFilesInPath(rootPath string, terragruntOptions *options.TerragruntOptions) ([]string, error) {
 	configFiles := []string{}
 
-	err := filepath.Walk(rootPath, func(path string, info os.FileInfo, err error) error {
+	err := util.WalkWithSymlinks(rootPath, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}

terraform/source.go

@@ -58,7 +58,7 @@ func (src Source) EncodeSourceVersion() (string, error) {
 		sourceHash := sha256.New()
 		sourceDir := filepath.Clean(src.CanonicalSourceURL.Path)
 
-		err := filepath.Walk(sourceDir, func(path string, info os.FileInfo, err error) error {
+		err := util.WalkWithSymlinks(sourceDir, func(path string, info os.FileInfo, err error) error {
 			if err != nil {
 				// If we've encountered an error while walking the tree, give up
 				return err

test/fixtures/stack/disjoint-symlinks/a/terragrunt.hcl

@@ -0,0 +1,3 @@
+terraform {
+  source = "../module"
+}

test/fixtures/stack/disjoint-symlinks/module/main.tf

@@ -0,0 +1,5 @@
+resource "null_resource" "a" {}
+
+output "a" {
+  value = null_resource.a.id
+}

test/integration_test.go

@@ -11,6 +11,7 @@ import (
 	"regexp"
 	"strings"
 	"testing"
+	"time"
 
 	runall "github.com/gruntwork-io/terragrunt/cli/commands/run-all"
 	"github.com/gruntwork-io/terragrunt/cli/commands/terraform"
@@ -43,6 +44,7 @@ const (
 	testFixtureDisabledModule                 = "fixtures/disabled/"
 	testFixtureDisabledPath                   = "fixtures/disabled-path/"
 	testFixtureDisjoint                       = "fixtures/stack/disjoint"
+	textFixtureDisjointSymlinks               = "fixtures/stack/disjoint-symlinks"
 	testFixtureDownload                       = "fixtures/download"
 	testFixtureEmptyState                     = "fixtures/empty-state/"
 	testFixtureEnvVarsBlockPath               = "fixtures/env-vars-block/"
@@ -637,6 +639,82 @@ func TestTerragruntStackCommandsWithPlanFile(t *testing.T) {
 	helpers.RunTerragrunt(t, "terragrunt apply-all plan.tfplan --terragrunt-log-level info --terragrunt-non-interactive --terragrunt-working-dir "+disjointEnvironmentPath)
 }
 
+func TestTerragruntStackCommandsWithSymlinks(t *testing.T) {
+	t.Parallel()
+
+	// please be aware that helpers.CopyEnvironment resolves symlinks statically,
+	// so the symlinked directories are copied physically, which defeats the purpose of this test,
+	// therefore we are going to create the symlinks manually in the destination directory
+	tmpEnvPath, err := filepath.EvalSymlinks(helpers.CopyEnvironment(t, textFixtureDisjointSymlinks))
+	require.NoError(t, err)
+	disjointSymlinksEnvironmentPath := util.JoinPath(tmpEnvPath, textFixtureDisjointSymlinks)
+	require.NoError(t, os.Symlink(util.JoinPath(disjointSymlinksEnvironmentPath, "a"), util.JoinPath(disjointSymlinksEnvironmentPath, "b")))
+	require.NoError(t, os.Symlink(util.JoinPath(disjointSymlinksEnvironmentPath, "a"), util.JoinPath(disjointSymlinksEnvironmentPath, "c")))
+
+	helpers.CleanupTerraformFolder(t, disjointSymlinksEnvironmentPath)
+
+	// perform the first initialization
+	_, stderr, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt run-all init --terragrunt-log-level info --terragrunt-non-interactive --terragrunt-working-dir "+disjointSymlinksEnvironmentPath)
+	require.NoError(t, err)
+	assert.Contains(t, stderr, "Downloading Terraform configurations from ./module into ./a/.terragrunt-cache")
+	assert.Contains(t, stderr, "Downloading Terraform configurations from ./module into ./b/.terragrunt-cache")
+	assert.Contains(t, stderr, "Downloading Terraform configurations from ./module into ./c/.terragrunt-cache")
+
+	// perform the second initialization and make sure that the cache is not downloaded again
+	_, stderr, err = helpers.RunTerragruntCommandWithOutput(t, "terragrunt run-all init --terragrunt-log-level info --terragrunt-non-interactive --terragrunt-working-dir "+disjointSymlinksEnvironmentPath)
+	require.NoError(t, err)
+	assert.NotContains(t, stderr, "Downloading Terraform configurations from ./module into ./a/.terragrunt-cache")
+	assert.NotContains(t, stderr, "Downloading Terraform configurations from ./module into ./b/.terragrunt-cache")
+	assert.NotContains(t, stderr, "Downloading Terraform configurations from ./module into ./c/.terragrunt-cache")
+
+	// validate the modules
+	_, stderr, err = helpers.RunTerragruntCommandWithOutput(t, "terragrunt run-all validate --terragrunt-log-level info --terragrunt-non-interactive --terragrunt-working-dir "+disjointSymlinksEnvironmentPath)
+	require.NoError(t, err)
+	assert.Contains(t, stderr, "Module ./a")
+	assert.Contains(t, stderr, "Module ./b")
+	assert.Contains(t, stderr, "Module ./c")
+
+	// touch the "module/main.tf" file to change the timestamp and make sure that the cache is downloaded again
+	require.NoError(t, os.Chtimes(util.JoinPath(disjointSymlinksEnvironmentPath, "module/main.tf"), time.Now(), time.Now()))
+
+	// perform the initialization and make sure that the cache is downloaded again
+	_, stderr, err = helpers.RunTerragruntCommandWithOutput(t, "terragrunt run-all init --terragrunt-log-level info --terragrunt-non-interactive --terragrunt-working-dir "+disjointSymlinksEnvironmentPath)
+	require.NoError(t, err)
+	assert.Contains(t, stderr, "Downloading Terraform configurations from ./module into ./a/.terragrunt-cache")
+	assert.Contains(t, stderr, "Downloading Terraform configurations from ./module into ./b/.terragrunt-cache")
+	assert.Contains(t, stderr, "Downloading Terraform configurations from ./module into ./c/.terragrunt-cache")
+}
+
+func TestTerragruntOutputModuleGroupsWithSymlinks(t *testing.T) {
+	t.Parallel()
+
+	// please be aware that helpers.CopyEnvironment resolves symlinks statically,
+	// so the symlinked directories are copied physically, which defeats the purpose of this test,
+	// therefore we are going to create the symlinks manually in the destination directory
+	tmpEnvPath, err := filepath.EvalSymlinks(helpers.CopyEnvironment(t, textFixtureDisjointSymlinks))
+	require.NoError(t, err)
+	disjointSymlinksEnvironmentPath := util.JoinPath(tmpEnvPath, textFixtureDisjointSymlinks)
+	require.NoError(t, os.Symlink(util.JoinPath(disjointSymlinksEnvironmentPath, "a"), util.JoinPath(disjointSymlinksEnvironmentPath, "b")))
+	require.NoError(t, os.Symlink(util.JoinPath(disjointSymlinksEnvironmentPath, "a"), util.JoinPath(disjointSymlinksEnvironmentPath, "c")))
+
+	expectedApplyOutput := fmt.Sprintf(`
+	{
+	  "Group 1": [
+		"%[1]s/a",
+		"%[1]s/b",
+		"%[1]s/c"
+	  ]
+	}`, disjointSymlinksEnvironmentPath)
+
+	helpers.CleanupTerraformFolder(t, disjointSymlinksEnvironmentPath)
+	stdout, _, err := helpers.RunTerragruntCommandWithOutput(t, fmt.Sprintf("terragrunt output-module-groups --terragrunt-working-dir %s apply", disjointSymlinksEnvironmentPath))
+	require.NoError(t, err)
+
+	output := strings.ReplaceAll(stdout, " ", "")
+	expectedOutput := strings.ReplaceAll(strings.ReplaceAll(expectedApplyOutput, "\t", ""), " ", "")
+	assert.True(t, strings.Contains(strings.TrimSpace(output), strings.TrimSpace(expectedOutput)))
+}
+
 func TestInvalidSource(t *testing.T) {
 	t.Parallel()
 

util/file.go

@@ -640,7 +640,7 @@ func (err PathIsNotFile) Error() string {
 func ListTfFiles(directoryPath string) ([]string, error) {
 	var tfFiles []string
 
-	err := filepath.Walk(directoryPath, func(path string, info os.FileInfo, err error) error {
+	err := WalkWithSymlinks(directoryPath, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -829,3 +829,95 @@ func Copy(ctx context.Context, dst io.Writer, src io.Reader) (int64, error) {
 
 	return num, err
 }
+
+// WalkWithSymlinks traverses a directory tree, following symbolic links and calling
+// the provided function for each file or directory encountered. It handles both regular
+// symlinks and circular symlinks without getting into infinite loops.
+//
+//nolint:funlen,gocognit,cyclop
+func WalkWithSymlinks(root string, externalWalkFn filepath.WalkFunc) error {
+	// pathPair keeps track of both the physical (real) path on disk
+	// and the logical path (how it appears in the walk)
+	type pathPair struct {
+		physical string
+		logical  string
+	}
+
+	// visited tracks symlink paths to prevent circular references
+	// key is combination of realPath:symlinkPath
+	visited := make(map[string]bool)
+
+	// visitedLogical tracks logical paths to prevent duplicates
+	// when the same directory is reached through different symlinks
+	visitedLogical := make(map[string]bool)
+
+	var walkFn func(pathPair) error
+
+	walkFn = func(pair pathPair) error {
+		return filepath.Walk(pair.physical, func(currentPath string, info os.FileInfo, err error) error {
+			if err != nil {
+				return externalWalkFn(currentPath, info, err)
+			}
+
+			// Convert the current physical path to a logical path relative to the walk root
+			rel, err := filepath.Rel(pair.physical, currentPath)
+			if err != nil {
+				return fmt.Errorf("failed to get relative path between %s and %s: %w", pair.physical, currentPath, err)
+			}
+
+			logicalPath := filepath.Join(pair.logical, rel)
+
+			// Call the provided function only if we haven't seen this logical path before
+			if !visitedLogical[logicalPath] {
+				visitedLogical[logicalPath] = true
+
+				if err := externalWalkFn(logicalPath, info, nil); err != nil {
+					return err
+				}
+			}
+
+			// If we encounter a symlink, resolve and follow it
+			if info.Mode()&os.ModeSymlink != 0 {
+				realPath, err := filepath.EvalSymlinks(currentPath)
+				if err != nil {
+					return fmt.Errorf("failed to get evalutate sym links for %s: %w", currentPath, err)
+				}
+
+				// Skip if we've seen this symlink->target combination before
+				// This prevents infinite loops with circular symlinks
+				if visited[realPath+":"+currentPath] {
+					return nil
+				}
+
+				visited[realPath+":"+currentPath] = true
+
+				// Get info about the symlink target
+				realInfo, err := os.Stat(realPath)
+				if err != nil {
+					return fmt.Errorf("failed to describe file %s: %w", realPath, err)
+				}
+
+				// If the target is a directory, recursively walk it
+				if realInfo.IsDir() {
+					return walkFn(pathPair{
+						physical: realPath,
+						logical:  logicalPath,
+					})
+				}
+			}
+
+			return nil
+		})
+	}
+
+	realRoot, err := filepath.EvalSymlinks(root)
+	if err != nil {
+		return fmt.Errorf("failed to get evalutate sym links for %s: %w", root, err)
+	}
+
+	// Start the walk from the root directory
+	return walkFn(pathPair{
+		physical: realRoot,
+		logical:  realRoot,
+	})
+}