From e86762fd4852c21dd2fc6f6b5c46c02bc24b88a9 Mon Sep 17 00:00:00 2001 From: celestialorb Date: Sat, 17 Apr 2021 13:29:40 +0000 Subject: [PATCH 1/7] reordering import --- config/config.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/config.go b/config/config.go index 31f1f4a9e..6c6947a63 100644 --- a/config/config.go +++ b/config/config.go @@ -2,12 +2,13 @@ package config import ( "fmt" - "github.com/mitchellh/mapstructure" "os" "path/filepath" "reflect" "strings" + "github.com/mitchellh/mapstructure" + "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclparse" "github.com/sirupsen/logrus" From 12fea1a23f6791c62e9260d474295d54f96aa861 Mon Sep 17 00:00:00 2001 From: celestialorb Date: Sat, 17 Apr 2021 13:29:51 +0000 Subject: [PATCH 2/7] simple test for finding config files with strict include --- config/config_test.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/config/config_test.go b/config/config_test.go index d0420a25a..af3fc24c5 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1066,6 +1066,25 @@ func TestFindConfigFilesInPathMultipleMixedConfigs(t *testing.T) { assert.Equal(t, expected, actual) } +func TestFindConfigFilesInPathStrictInclude(t *testing.T) { + t.Parallel() + + expected := []string{ + "../test/fixture-config-files/multiple-mixed-configs/subdir-2/subdir/terragrunt.hcl", + } + + terragruntOptions, err := options.NewTerragruntOptionsForTest("test") + require.NoError(t, err) + + terragruntOptions.StrictInclude = true + terragruntOptions.IncludeDirs = []string{"subdir-2"} + + actual, err := FindConfigFilesInPath("../test/fixture-config-files/multiple-mixed-configs", terragruntOptions) + assert.Nil(t, err, "Unexpected error: %v", err) + + assert.Equal(t, expected, actual) +} + func TestFindConfigFilesIgnoresTerragruntCache(t *testing.T) { t.Parallel() From ddf8db6a58826e29ce1cdb637f4927a0242ad5b7 Mon Sep 17 00:00:00 2001 From: celestialorb Date: Sat, 17 Apr 2021 13:30:07 +0000 Subject: [PATCH 3/7] restricting directories walked over with strict include --- config/config.go | 55 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/config/config.go b/config/config.go index 6c6947a63..e43875895 100644 --- a/config/config.go +++ b/config/config.go @@ -352,29 +352,50 @@ func GetDefaultConfigPath(workingDir string) string { func FindConfigFilesInPath(rootPath string, terragruntOptions *options.TerragruntOptions) ([]string, error) { configFiles := []string{} - err := filepath.Walk(rootPath, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err + // Determine the set of file paths that we need to walk over, if strict include is not set default to walk over the + // entire root path (working directory). + directories := []string{rootPath} + + // If struct include is set, rebuild the set of directories to only include those specified by the Terragrunt + // include dir flags. + if terragruntOptions.StrictInclude { + directories = []string{} + for _, directory := range terragruntOptions.IncludeDirs { + directories = append(directories, filepath.Join(rootPath, directory)) } + } - // Skip the Terragrunt cache dir entirely - if info.IsDir() && info.Name() == options.TerragruntCacheDir { - return filepath.SkipDir - } + for _, directory := range directories { + terragruntOptions.Logger.Logf(logrus.InfoLevel, "walking over directory for terragrunt modules: %s", directory) + err := filepath.Walk(directory, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } - isTerragruntModule, err := containsTerragruntModule(path, info, terragruntOptions) - if err != nil { - return err - } + // Skip the Terragrunt cache dir entirely + if info.IsDir() && info.Name() == options.TerragruntCacheDir { + return filepath.SkipDir + } - if isTerragruntModule { - configFiles = append(configFiles, GetDefaultConfigPath(path)) - } + isTerragruntModule, err := containsTerragruntModule(path, info, terragruntOptions) + if err != nil { + return err + } - return nil - }) + if isTerragruntModule { + terragruntOptions.Logger.Logf(logrus.InfoLevel, "found terragrunt module: %s", path) + configFiles = append(configFiles, GetDefaultConfigPath(path)) + } + + return nil + }) + + if err != nil { + return configFiles, err + } + } - return configFiles, err + return configFiles, nil } // Returns true if the given path with the given FileInfo contains a Terragrunt module and false otherwise. A path From 3f22af8a8598bdae9ed27943e65e936363619452 Mon Sep 17 00:00:00 2001 From: celestialorb Date: Sat, 17 Apr 2021 13:59:00 +0000 Subject: [PATCH 4/7] reducing module walk for just include dir flags --- config/config.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/config/config.go b/config/config.go index e43875895..2e8152f1b 100644 --- a/config/config.go +++ b/config/config.go @@ -352,17 +352,15 @@ func GetDefaultConfigPath(workingDir string) string { func FindConfigFilesInPath(rootPath string, terragruntOptions *options.TerragruntOptions) ([]string, error) { configFiles := []string{} - // Determine the set of file paths that we need to walk over, if strict include is not set default to walk over the - // entire root path (working directory). + // Start from the given root path, unless we're given specific directories to include or strict include is on. directories := []string{rootPath} - - // If struct include is set, rebuild the set of directories to only include those specified by the Terragrunt - // include dir flags. - if terragruntOptions.StrictInclude { + if terragruntOptions.StrictInclude || len(terragruntOptions.IncludeDirs) > 0 { directories = []string{} - for _, directory := range terragruntOptions.IncludeDirs { - directories = append(directories, filepath.Join(rootPath, directory)) - } + } + + // Build a collection of directories based off of any included directories from the given terragrunt options. + for _, directory := range terragruntOptions.IncludeDirs { + directories = append(directories, filepath.Join(rootPath, directory)) } for _, directory := range directories { From c7339c81f3633cdc69d942684e031912b1ad4434 Mon Sep 17 00:00:00 2001 From: celestialorb Date: Sat, 17 Apr 2021 14:00:31 +0000 Subject: [PATCH 5/7] adding test for just include dirs --- config/config_test.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/config/config_test.go b/config/config_test.go index af3fc24c5..81e67e883 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1066,6 +1066,26 @@ func TestFindConfigFilesInPathMultipleMixedConfigs(t *testing.T) { assert.Equal(t, expected, actual) } +func TestFindConfigFilesInPathIncludeDirs(t *testing.T) { + t.Parallel() + + expected := []string{ + "../test/fixture-config-files/multiple-mixed-configs/subdir-2/subdir/terragrunt.hcl", + "../test/fixture-config-files/multiple-mixed-configs/subdir-3/terragrunt.hcl.json", + } + + terragruntOptions, err := options.NewTerragruntOptionsForTest("test") + require.NoError(t, err) + + terragruntOptions.StrictInclude = true + terragruntOptions.IncludeDirs = []string{"subdir-2", "subdir-3"} + + actual, err := FindConfigFilesInPath("../test/fixture-config-files/multiple-mixed-configs", terragruntOptions) + assert.Nil(t, err, "Unexpected error: %v", err) + + assert.Equal(t, expected, actual) +} + func TestFindConfigFilesInPathStrictInclude(t *testing.T) { t.Parallel() From b4c91e5a0714c8a99c5c38886734b85416ecf673 Mon Sep 17 00:00:00 2001 From: celestialorb Date: Thu, 29 Apr 2021 18:05:38 +0000 Subject: [PATCH 6/7] adding basic glob test --- config/config_test.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/config/config_test.go b/config/config_test.go index 81e67e883..0b17bcf13 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1215,6 +1215,26 @@ func TestFindConfigFilesIgnoresDownloadDir(t *testing.T) { assert.Equal(t, expected, actual) } +func TestFindConfigFilesInPathWithGlob(t *testing.T) { + t.Parallel() + + expected := []string{ + "../test/fixture-config-files/multiple-mixed-configs/subdir-2/subdir/terragrunt.hcl", + "../test/fixture-config-files/multiple-mixed-configs/subdir-3/terragrunt.hcl.json", + } + + terragruntOptions, err := options.NewTerragruntOptionsForTest("test") + require.NoError(t, err) + + terragruntOptions.StrictInclude = true + terragruntOptions.IncludeDirs = []string{"subdir-*"} + + actual, err := FindConfigFilesInPath("../test/fixture-config-files/multiple-mixed-configs", terragruntOptions) + assert.Nil(t, err, "Unexpected error: %v", err) + + assert.Equal(t, expected, actual) +} + func mockOptionsForTestWithConfigPath(t *testing.T, configPath string) *options.TerragruntOptions { opts, err := options.NewTerragruntOptionsForTest(configPath) if err != nil { From a1be5a0ac4419cd910d6326700e0be110d70dd1f Mon Sep 17 00:00:00 2001 From: celestialorb Date: Thu, 29 Apr 2021 18:05:50 +0000 Subject: [PATCH 7/7] adding implementation and reordering expected results --- config/config.go | 16 +++++++++++++++- config/config_test.go | 24 ++++++++++++------------ 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/config/config.go b/config/config.go index 2e8152f1b..a3c783e9a 100644 --- a/config/config.go +++ b/config/config.go @@ -5,8 +5,10 @@ import ( "os" "path/filepath" "reflect" + "sort" "strings" + zglob "github.com/mattn/go-zglob" "github.com/mitchellh/mapstructure" "github.com/hashicorp/hcl/v2" @@ -360,9 +362,17 @@ func FindConfigFilesInPath(rootPath string, terragruntOptions *options.Terragrun // Build a collection of directories based off of any included directories from the given terragrunt options. for _, directory := range terragruntOptions.IncludeDirs { - directories = append(directories, filepath.Join(rootPath, directory)) + matches, err := zglob.Glob(filepath.Join(rootPath, directory)) + if err != nil { + return nil, err + } + + directories = append(directories, matches...) } + // TODO: since the terragrunt include directories can be specified as globs, we need to expand + // those globs before walking over them. + for _, directory := range directories { terragruntOptions.Logger.Logf(logrus.InfoLevel, "walking over directory for terragrunt modules: %s", directory) err := filepath.Walk(directory, func(path string, info os.FileInfo, err error) error { @@ -393,6 +403,10 @@ func FindConfigFilesInPath(rootPath string, terragruntOptions *options.Terragrun } } + // NOTE: we're sorting the configuration files as I found that very rarely the order returned by zglob wasn't + // always determinate. I don't know if this is a bug in the zglob module or just side-effect of a race in the module. + // @celestialorb, 2021/04/29 + sort.Strings(configFiles) return configFiles, nil } diff --git a/config/config_test.go b/config/config_test.go index 0b17bcf13..6e31d3eb5 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1019,9 +1019,9 @@ func TestFindConfigFilesInPathMultipleConfigs(t *testing.T) { t.Parallel() expected := []string{ - "../test/fixture-config-files/multiple-configs/terragrunt.hcl", "../test/fixture-config-files/multiple-configs/subdir-2/subdir/terragrunt.hcl", "../test/fixture-config-files/multiple-configs/subdir-3/terragrunt.hcl", + "../test/fixture-config-files/multiple-configs/terragrunt.hcl", } terragruntOptions, err := options.NewTerragruntOptionsForTest("test") require.NoError(t, err) @@ -1036,9 +1036,9 @@ func TestFindConfigFilesInPathMultipleJsonConfigs(t *testing.T) { t.Parallel() expected := []string{ - "../test/fixture-config-files/multiple-json-configs/terragrunt.hcl.json", "../test/fixture-config-files/multiple-json-configs/subdir-2/subdir/terragrunt.hcl.json", "../test/fixture-config-files/multiple-json-configs/subdir-3/terragrunt.hcl.json", + "../test/fixture-config-files/multiple-json-configs/terragrunt.hcl.json", } terragruntOptions, err := options.NewTerragruntOptionsForTest("test") require.NoError(t, err) @@ -1053,9 +1053,9 @@ func TestFindConfigFilesInPathMultipleMixedConfigs(t *testing.T) { t.Parallel() expected := []string{ - "../test/fixture-config-files/multiple-mixed-configs/terragrunt.hcl.json", "../test/fixture-config-files/multiple-mixed-configs/subdir-2/subdir/terragrunt.hcl", "../test/fixture-config-files/multiple-mixed-configs/subdir-3/terragrunt.hcl.json", + "../test/fixture-config-files/multiple-mixed-configs/terragrunt.hcl.json", } terragruntOptions, err := options.NewTerragruntOptionsForTest("test") require.NoError(t, err) @@ -1124,10 +1124,10 @@ func TestFindConfigFilesIgnoresTerraformDataDir(t *testing.T) { t.Parallel() expected := []string{ - "../test/fixture-config-files/ignore-terraform-data-dir/terragrunt.hcl", "../test/fixture-config-files/ignore-terraform-data-dir/.tf_data/modules/mod/terragrunt.hcl", - "../test/fixture-config-files/ignore-terraform-data-dir/subdir/terragrunt.hcl", "../test/fixture-config-files/ignore-terraform-data-dir/subdir/.tf_data/modules/mod/terragrunt.hcl", + "../test/fixture-config-files/ignore-terraform-data-dir/subdir/terragrunt.hcl", + "../test/fixture-config-files/ignore-terraform-data-dir/terragrunt.hcl", } terragruntOptions, err := options.NewTerragruntOptionsForTest("test") require.NoError(t, err) @@ -1142,9 +1142,9 @@ func TestFindConfigFilesIgnoresTerraformDataDirEnv(t *testing.T) { t.Parallel() expected := []string{ - "../test/fixture-config-files/ignore-terraform-data-dir/terragrunt.hcl", - "../test/fixture-config-files/ignore-terraform-data-dir/subdir/terragrunt.hcl", "../test/fixture-config-files/ignore-terraform-data-dir/subdir/.terraform/modules/mod/terragrunt.hcl", + "../test/fixture-config-files/ignore-terraform-data-dir/subdir/terragrunt.hcl", + "../test/fixture-config-files/ignore-terraform-data-dir/terragrunt.hcl", } terragruntOptions, err := options.NewTerragruntOptionsForTest("test") require.NoError(t, err) @@ -1160,10 +1160,10 @@ func TestFindConfigFilesIgnoresTerraformDataDirEnvPath(t *testing.T) { t.Parallel() expected := []string{ - "../test/fixture-config-files/ignore-terraform-data-dir/terragrunt.hcl", "../test/fixture-config-files/ignore-terraform-data-dir/.tf_data/modules/mod/terragrunt.hcl", - "../test/fixture-config-files/ignore-terraform-data-dir/subdir/terragrunt.hcl", "../test/fixture-config-files/ignore-terraform-data-dir/subdir/.terraform/modules/mod/terragrunt.hcl", + "../test/fixture-config-files/ignore-terraform-data-dir/subdir/terragrunt.hcl", + "../test/fixture-config-files/ignore-terraform-data-dir/terragrunt.hcl", } terragruntOptions, err := options.NewTerragruntOptionsForTest("test") require.NoError(t, err) @@ -1182,10 +1182,10 @@ func TestFindConfigFilesIgnoresTerraformDataDirEnvRoot(t *testing.T) { require.NoError(t, err) expected := []string{ - filepath.Join(cwd, "../test/fixture-config-files/ignore-terraform-data-dir/terragrunt.hcl"), - filepath.Join(cwd, "../test/fixture-config-files/ignore-terraform-data-dir/subdir/terragrunt.hcl"), filepath.Join(cwd, "../test/fixture-config-files/ignore-terraform-data-dir/subdir/.terraform/modules/mod/terragrunt.hcl"), filepath.Join(cwd, "../test/fixture-config-files/ignore-terraform-data-dir/subdir/.tf_data/modules/mod/terragrunt.hcl"), + filepath.Join(cwd, "../test/fixture-config-files/ignore-terraform-data-dir/subdir/terragrunt.hcl"), + filepath.Join(cwd, "../test/fixture-config-files/ignore-terraform-data-dir/terragrunt.hcl"), } workingDir := filepath.Join(cwd, "../test/fixture-config-files/ignore-terraform-data-dir/") terragruntOptions, err := options.NewTerragruntOptionsForTest(workingDir) @@ -1202,8 +1202,8 @@ func TestFindConfigFilesIgnoresDownloadDir(t *testing.T) { t.Parallel() expected := []string{ - "../test/fixture-config-files/multiple-configs/terragrunt.hcl", "../test/fixture-config-files/multiple-configs/subdir-3/terragrunt.hcl", + "../test/fixture-config-files/multiple-configs/terragrunt.hcl", } terragruntOptions, err := options.NewTerragruntOptionsForTest("test") require.NoError(t, err)