From 8f89e6765cf7be01d79492c7a92eb92f48bd5dda Mon Sep 17 00:00:00 2001 From: Rodrigo Fior Kuntzer Date: Tue, 19 Nov 2024 18:14:08 +0100 Subject: [PATCH] feat: support discovery of symlinked modules Signed-off-by: Rodrigo Fior Kuntzer --- cli/commands/catalog/module/repo.go | 4 +- config/config.go | 2 +- terraform/source.go | 2 +- .../stack/disjoint-symlinks/a/terragrunt.hcl | 3 + .../stack/disjoint-symlinks/module/main.tf | 5 + test/integration_test.go | 78 ++++++++ util/file.go | 103 ++++++++++- util/file_test.go | 173 ++++++++++++++++++ 8 files changed, 366 insertions(+), 4 deletions(-) create mode 100644 test/fixtures/stack/disjoint-symlinks/a/terragrunt.hcl create mode 100644 test/fixtures/stack/disjoint-symlinks/module/main.tf diff --git a/cli/commands/catalog/module/repo.go b/cli/commands/catalog/module/repo.go index e25efef3ae..22cc3abc45 100644 --- a/cli/commands/catalog/module/repo.go +++ b/cli/commands/catalog/module/repo.go @@ -9,6 +9,8 @@ import ( "regexp" "strings" + "github.com/gruntwork-io/terragrunt/util" + "github.com/gitsight/go-vcsurl" "github.com/gruntwork-io/go-commons/files" "github.com/gruntwork-io/terragrunt/internal/errors" @@ -82,7 +84,7 @@ func (repo *Repo) FindModules(ctx context.Context) (Modules, error) { continue } - err := filepath.Walk(modulesPath, + err := util.WalkWithSymlinks(modulesPath, func(dir string, remote os.FileInfo, err error) error { if err != nil { return err diff --git a/config/config.go b/config/config.go index e54e9218fc..b531e4aa23 100644 --- a/config/config.go +++ b/config/config.go @@ -650,7 +650,7 @@ func GetDefaultConfigPath(workingDir string) string { func FindConfigFilesInPath(rootPath string, terragruntOptions *options.TerragruntOptions) ([]string, error) { configFiles := []string{} - err := filepath.Walk(rootPath, func(path string, info os.FileInfo, err error) error { + err := util.WalkWithSymlinks(rootPath, func(path string, info os.FileInfo, err error) error { if err != nil { return err } diff --git a/terraform/source.go b/terraform/source.go index be07e2566b..8df0d940b8 100644 --- a/terraform/source.go +++ b/terraform/source.go @@ -58,7 +58,7 @@ func (src Source) EncodeSourceVersion() (string, error) { sourceHash := sha256.New() sourceDir := filepath.Clean(src.CanonicalSourceURL.Path) - err := filepath.Walk(sourceDir, func(path string, info os.FileInfo, err error) error { + err := util.WalkWithSymlinks(sourceDir, func(path string, info os.FileInfo, err error) error { if err != nil { // If we've encountered an error while walking the tree, give up return err diff --git a/test/fixtures/stack/disjoint-symlinks/a/terragrunt.hcl b/test/fixtures/stack/disjoint-symlinks/a/terragrunt.hcl new file mode 100644 index 0000000000..8628984b0c --- /dev/null +++ b/test/fixtures/stack/disjoint-symlinks/a/terragrunt.hcl @@ -0,0 +1,3 @@ +terraform { + source = "../module" +} diff --git a/test/fixtures/stack/disjoint-symlinks/module/main.tf b/test/fixtures/stack/disjoint-symlinks/module/main.tf new file mode 100644 index 0000000000..8c31a965e1 --- /dev/null +++ b/test/fixtures/stack/disjoint-symlinks/module/main.tf @@ -0,0 +1,5 @@ +resource "null_resource" "a" {} + +output "a" { + value = null_resource.a.id +} diff --git a/test/integration_test.go b/test/integration_test.go index 7159920948..5d6049af49 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -11,6 +11,7 @@ import ( "regexp" "strings" "testing" + "time" runall "github.com/gruntwork-io/terragrunt/cli/commands/run-all" "github.com/gruntwork-io/terragrunt/cli/commands/terraform" @@ -43,6 +44,7 @@ const ( testFixtureDisabledModule = "fixtures/disabled/" testFixtureDisabledPath = "fixtures/disabled-path/" testFixtureDisjoint = "fixtures/stack/disjoint" + textFixtureDisjointSymlinks = "fixtures/stack/disjoint-symlinks" testFixtureDownload = "fixtures/download" testFixtureEmptyState = "fixtures/empty-state/" testFixtureEnvVarsBlockPath = "fixtures/env-vars-block/" @@ -637,6 +639,82 @@ func TestTerragruntStackCommandsWithPlanFile(t *testing.T) { helpers.RunTerragrunt(t, "terragrunt apply-all plan.tfplan --terragrunt-log-level info --terragrunt-non-interactive --terragrunt-working-dir "+disjointEnvironmentPath) } +func TestTerragruntStackCommandsWithSymlinks(t *testing.T) { + t.Parallel() + + // please be aware that helpers.CopyEnvironment resolves symlinks statically, + // so the symlinked directories are copied physically, which defeats the purpose of this test, + // therefore we are going to create the symlinks manually in the destination directory + tmpEnvPath, err := filepath.EvalSymlinks(helpers.CopyEnvironment(t, textFixtureDisjointSymlinks)) + require.NoError(t, err) + disjointSymlinksEnvironmentPath := util.JoinPath(tmpEnvPath, textFixtureDisjointSymlinks) + require.NoError(t, os.Symlink(util.JoinPath(disjointSymlinksEnvironmentPath, "a"), util.JoinPath(disjointSymlinksEnvironmentPath, "b"))) + require.NoError(t, os.Symlink(util.JoinPath(disjointSymlinksEnvironmentPath, "a"), util.JoinPath(disjointSymlinksEnvironmentPath, "c"))) + + helpers.CleanupTerraformFolder(t, disjointSymlinksEnvironmentPath) + + // perform the first initialization + _, stderr, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt run-all init --terragrunt-log-level info --terragrunt-non-interactive --terragrunt-working-dir "+disjointSymlinksEnvironmentPath) + require.NoError(t, err) + assert.Contains(t, stderr, "Downloading Terraform configurations from ./module into ./a/.terragrunt-cache") + assert.Contains(t, stderr, "Downloading Terraform configurations from ./module into ./b/.terragrunt-cache") + assert.Contains(t, stderr, "Downloading Terraform configurations from ./module into ./c/.terragrunt-cache") + + // perform the second initialization and make sure that the cache is not downloaded again + _, stderr, err = helpers.RunTerragruntCommandWithOutput(t, "terragrunt run-all init --terragrunt-log-level info --terragrunt-non-interactive --terragrunt-working-dir "+disjointSymlinksEnvironmentPath) + require.NoError(t, err) + assert.NotContains(t, stderr, "Downloading Terraform configurations from ./module into ./a/.terragrunt-cache") + assert.NotContains(t, stderr, "Downloading Terraform configurations from ./module into ./b/.terragrunt-cache") + assert.NotContains(t, stderr, "Downloading Terraform configurations from ./module into ./c/.terragrunt-cache") + + // validate the modules + _, stderr, err = helpers.RunTerragruntCommandWithOutput(t, "terragrunt run-all validate --terragrunt-log-level info --terragrunt-non-interactive --terragrunt-working-dir "+disjointSymlinksEnvironmentPath) + require.NoError(t, err) + assert.Contains(t, stderr, "Module ./a") + assert.Contains(t, stderr, "Module ./b") + assert.Contains(t, stderr, "Module ./c") + + // touch the "module/main.tf" file to change the timestamp and make sure that the cache is downloaded again + require.NoError(t, os.Chtimes(util.JoinPath(disjointSymlinksEnvironmentPath, "module/main.tf"), time.Now(), time.Now())) + + // perform the initialization and make sure that the cache is downloaded again + _, stderr, err = helpers.RunTerragruntCommandWithOutput(t, "terragrunt run-all init --terragrunt-log-level info --terragrunt-non-interactive --terragrunt-working-dir "+disjointSymlinksEnvironmentPath) + require.NoError(t, err) + assert.Contains(t, stderr, "Downloading Terraform configurations from ./module into ./a/.terragrunt-cache") + assert.Contains(t, stderr, "Downloading Terraform configurations from ./module into ./b/.terragrunt-cache") + assert.Contains(t, stderr, "Downloading Terraform configurations from ./module into ./c/.terragrunt-cache") +} + +func TestTerragruntOutputModuleGroupsWithSymlinks(t *testing.T) { + t.Parallel() + + // please be aware that helpers.CopyEnvironment resolves symlinks statically, + // so the symlinked directories are copied physically, which defeats the purpose of this test, + // therefore we are going to create the symlinks manually in the destination directory + tmpEnvPath, err := filepath.EvalSymlinks(helpers.CopyEnvironment(t, textFixtureDisjointSymlinks)) + require.NoError(t, err) + disjointSymlinksEnvironmentPath := util.JoinPath(tmpEnvPath, textFixtureDisjointSymlinks) + require.NoError(t, os.Symlink(util.JoinPath(disjointSymlinksEnvironmentPath, "a"), util.JoinPath(disjointSymlinksEnvironmentPath, "b"))) + require.NoError(t, os.Symlink(util.JoinPath(disjointSymlinksEnvironmentPath, "a"), util.JoinPath(disjointSymlinksEnvironmentPath, "c"))) + + expectedApplyOutput := fmt.Sprintf(` + { + "Group 1": [ + "%[1]s/a", + "%[1]s/b", + "%[1]s/c" + ] + }`, disjointSymlinksEnvironmentPath) + + helpers.CleanupTerraformFolder(t, disjointSymlinksEnvironmentPath) + stdout, _, err := helpers.RunTerragruntCommandWithOutput(t, fmt.Sprintf("terragrunt output-module-groups --terragrunt-working-dir %s apply", disjointSymlinksEnvironmentPath)) + require.NoError(t, err) + + output := strings.ReplaceAll(stdout, " ", "") + expectedOutput := strings.ReplaceAll(strings.ReplaceAll(expectedApplyOutput, "\t", ""), " ", "") + assert.True(t, strings.Contains(strings.TrimSpace(output), strings.TrimSpace(expectedOutput))) +} + func TestInvalidSource(t *testing.T) { t.Parallel() diff --git a/util/file.go b/util/file.go index f907ebf64e..076530b966 100644 --- a/util/file.go +++ b/util/file.go @@ -640,7 +640,7 @@ func (err PathIsNotFile) Error() string { func ListTfFiles(directoryPath string) ([]string, error) { var tfFiles []string - err := filepath.Walk(directoryPath, func(path string, info os.FileInfo, err error) error { + err := WalkWithSymlinks(directoryPath, func(path string, info os.FileInfo, err error) error { if err != nil { return err } @@ -829,3 +829,104 @@ func Copy(ctx context.Context, dst io.Writer, src io.Reader) (int64, error) { return num, err } + +// WalkWithSymlinks traverses a directory tree, following symbolic links and calling +// the provided function for each file or directory encountered. It handles both regular +// symlinks and circular symlinks without getting into infinite loops. +// +//nolint:funlen +func WalkWithSymlinks(root string, externalWalkFn filepath.WalkFunc) error { + // pathPair keeps track of both the physical (real) path on disk + // and the logical path (how it appears in the walk) + type pathPair struct { + physical string + logical string + } + + // visited tracks symlink paths to prevent circular references + // key is combination of realPath:symlinkPath + visited := make(map[string]bool) + + // visitedLogical tracks logical paths to prevent duplicates + // when the same directory is reached through different symlinks + visitedLogical := make(map[string]bool) + + var walkFn func(pathPair) error + + walkFn = func(pair pathPair) error { + return filepath.Walk(pair.physical, func(currentPath string, info os.FileInfo, err error) error { + if err != nil { + return externalWalkFn(currentPath, info, err) + } + + // Convert the current physical path to a logical path relative to the walk root + rel, err := filepath.Rel(pair.physical, currentPath) + if err != nil { + return fmt.Errorf("failed to get relative path between %s and %s: %w", pair.physical, currentPath, err) + } + + logicalPath := filepath.Join(pair.logical, rel) + + realPath, realInfo, err := evalRealPathAndInfo(currentPath) + if err != nil { + return err + } + + // Call the provided function only if we haven't seen this logical path before + if !visitedLogical[logicalPath] { + visitedLogical[logicalPath] = true + + if err := externalWalkFn(logicalPath, realInfo, nil); err != nil { + return err + } + } + + // If we encounter a symlink, resolve and follow it + if info.Mode()&os.ModeSymlink != 0 { + // Skip if we've seen this symlink->target combination before + // This prevents infinite loops with circular symlinks + if visited[realPath+":"+currentPath] { + return nil + } + + visited[realPath+":"+currentPath] = true + + // If the target is a directory, recursively walk it + if realInfo.IsDir() { + return walkFn(pathPair{ + physical: realPath, + logical: logicalPath, + }) + } + } + + return nil + }) + } + + realRoot, err := filepath.EvalSymlinks(root) + if err != nil { + return fmt.Errorf("failed to get evaluate sym links for %s: %w", root, err) + } + + // Start the walk from the root directory + return walkFn(pathPair{ + physical: realRoot, + logical: realRoot, + }) +} + +func evalRealPathAndInfo(currentPath string) (string, os.FileInfo, error) { + realPath, err := filepath.EvalSymlinks(currentPath) + if err != nil { + return "", nil, fmt.Errorf("failed to get evaluate sym links for %s: %w", currentPath, err) + } + + // Get info about the symlink target + realInfo, err := os.Stat(realPath) + if err != nil { + return "", nil, fmt.Errorf("failed to describe file %s: %w", realPath, err) + } + + return realPath, realInfo, nil +} diff --git a/util/file_test.go b/util/file_test.go index d3d724dfa6..4aef2f1827 100644 --- a/util/file_test.go +++ b/util/file_test.go @@ -400,3 +400,176 @@ func TestEmptyDir(t *testing.T) { }) } } + +//nolint:funlen +func TestWalkWithSimpleSymlinks(t *testing.T) { + t.Parallel() + // Create temporary test directory structure + tempDir := t.TempDir() + tempDir, err := filepath.EvalSymlinks(tempDir) + require.NoError(t, err) + + // Create directories + dirs := []string{"a", "d"} + for _, dir := range dirs { + require.NoError(t, os.Mkdir(filepath.Join(tempDir, dir), 0755)) + } + + // Create test files + testFile := filepath.Join(tempDir, "a", "test.txt") + require.NoError(t, os.WriteFile(testFile, []byte("test"), 0644)) + + // Create symlinks + require.NoError(t, os.Symlink(filepath.Join(tempDir, "a"), filepath.Join(tempDir, "b"))) + require.NoError(t, os.Symlink(filepath.Join(tempDir, "a"), filepath.Join(tempDir, "c"))) + require.NoError(t, os.Symlink(filepath.Join(tempDir, "a"), filepath.Join(tempDir, "d", "a"))) + + var paths []string + err = util.WalkWithSymlinks(tempDir, func(path string, _ os.FileInfo, err error) error { + if err != nil { + return err + } + relPath, err := filepath.Rel(tempDir, path) + if err != nil { + t.Fatal(err) + } + paths = append(paths, relPath) + + return nil + }) + require.NoError(t, err) + + // Sort paths for reliable comparison + sort.Strings(paths) + + // Expected paths should include original and symlinked locations + expectedPaths := []string{ + ".", + "a", + "a/test.txt", + "b", + "b/test.txt", + "c", + "c/test.txt", + "d", + "d/a", + "d/a/test.txt", + } + sort.Strings(expectedPaths) + + if len(paths) != len(expectedPaths) { + t.Errorf("Got %d paths, expected %d", len(paths), len(expectedPaths)) + } + + for expectedPath := range expectedPaths { + if expectedPath >= len(paths) { + t.Errorf("Missing expected path: %s", expectedPaths[expectedPath]) + + continue + } + if paths[expectedPath] != expectedPaths[expectedPath] { + t.Errorf("Path mismatch at index %d:\ngot: %s\nwant: %s", expectedPath, paths[expectedPath], expectedPaths[expectedPath]) + } + } +} + +//nolint:funlen +func TestWalkWithCircularSymlinks(t *testing.T) { + t.Parallel() + // Create temporary test directory structure + tempDir := t.TempDir() + tempDir, err := filepath.EvalSymlinks(tempDir) + require.NoError(t, err) + + // Create directories + dirs := []string{"a", "b", "c", "d"} + for _, dir := range dirs { + require.NoError(t, os.Mkdir(filepath.Join(tempDir, dir), 0755)) + } + + // Create test files + testFile := filepath.Join(tempDir, "a", "test.txt") + require.NoError(t, os.WriteFile(testFile, []byte("test"), 0644)) + + // Create symlinks + require.NoError(t, os.Symlink(filepath.Join(tempDir, "a"), filepath.Join(tempDir, "b", "link-to-a"))) + require.NoError(t, os.Symlink(filepath.Join(tempDir, "a"), filepath.Join(tempDir, "c", "another-link-to-a"))) + + // Create circular symlink + require.NoError(t, os.Symlink(filepath.Join(tempDir, "d"), filepath.Join(tempDir, "a", "link-to-d"))) + require.NoError(t, os.Symlink(filepath.Join(tempDir, "a"), filepath.Join(tempDir, "d", "link-to-a"))) + + var paths []string + err = util.WalkWithSymlinks(tempDir, func(path string, _ os.FileInfo, err error) error { + if err != nil { + return err + } + relPath, err := filepath.Rel(tempDir, path) + if err != nil { + t.Fatal(err) + } + paths = append(paths, relPath) + + return nil + }) + require.NoError(t, err) + + // Sort paths for reliable comparison + sort.Strings(paths) + + // Expected paths should include original and symlinked locations + expectedPaths := []string{ + ".", + "a", + "a/link-to-d", + "a/link-to-d/link-to-a", + "a/link-to-d/link-to-a/link-to-d", + "a/link-to-d/link-to-a/test.txt", + "a/test.txt", + "b", + "b/link-to-a", + "b/link-to-a/link-to-d", + "b/link-to-a/test.txt", + "c", + "c/another-link-to-a", + "c/another-link-to-a/link-to-d", + "c/another-link-to-a/test.txt", + "d", + "d/link-to-a", + } + sort.Strings(expectedPaths) + + if len(paths) != len(expectedPaths) { + t.Errorf("Got %d paths, expected %d", len(paths), len(expectedPaths)) + } + + for expectedPath := range expectedPaths { + if expectedPath >= len(paths) { + t.Errorf("Missing expected path: %s", expectedPaths[expectedPath]) + + continue + } + if paths[expectedPath] != expectedPaths[expectedPath] { + t.Errorf("Path mismatch at index %d:\ngot: %s\nwant: %s", expectedPath, paths[expectedPath], expectedPaths[expectedPath]) + } + } +} + +func TestWalkWithSymlinksErrors(t *testing.T) { + t.Parallel() + + tempDir := t.TempDir() + + // Test with non-existent directory + require.Error(t, util.WalkWithSymlinks(filepath.Join(tempDir, "nonexistent"), func(_ string, _ os.FileInfo, err error) error { + return err + })) + + // Test with broken symlink + brokenLink := filepath.Join(tempDir, "broken") + require.NoError(t, os.Symlink(filepath.Join(tempDir, "nonexistent"), brokenLink)) + + require.Error(t, util.WalkWithSymlinks(tempDir, func(_ string, _ os.FileInfo, err error) error { + return err + })) +}