Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Restrict Locals Evaluation with Include Dirs #1644

Draft
wants to merge 8 commits into
base: main
Choose a base branch
from
Draft

Restrict Locals Evaluation with Include Dirs #1644

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
d8f13de
Merge pull request #1 from gruntwork-io/master
celestialorb Apr 17, 2021
e86762f
reordering import
celestialorb Apr 17, 2021
12fea1a
simple test for finding config files with strict include
celestialorb Apr 17, 2021
ddf8db6
restricting directories walked over with strict include
celestialorb Apr 17, 2021
3f22af8
reducing module walk for just include dir flags
celestialorb Apr 17, 2021
c7339c8
adding test for just include dirs
celestialorb Apr 17, 2021
b4c91e5
adding basic glob test
celestialorb Apr 29, 2021
a1be5a0
adding implementation and reordering expected results
celestialorb Apr 29, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 51 additions & 17 deletions config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,15 @@ package config

import (
"fmt"
"github.com/mitchellh/mapstructure"
"os"
"path/filepath"
"reflect"
"sort"
"strings"

zglob "github.com/mattn/go-zglob"
"github.com/mitchellh/mapstructure"

"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/hclparse"
"github.com/sirupsen/logrus"
Expand Down Expand Up @@ -351,29 +354,60 @@ func GetDefaultConfigPath(workingDir string) string {
func FindConfigFilesInPath(rootPath string, terragruntOptions *options.TerragruntOptions) ([]string, error) {
configFiles := []string{}

err := filepath.Walk(rootPath, func(path string, info os.FileInfo, err error) error {
// Start from the given root path, unless we're given specific directories to include or strict include is on.
directories := []string{rootPath}
if terragruntOptions.StrictInclude || len(terragruntOptions.IncludeDirs) > 0 {
directories = []string{}
}

// Build a collection of directories based off of any included directories from the given terragrunt options.
for _, directory := range terragruntOptions.IncludeDirs {
matches, err := zglob.Glob(filepath.Join(rootPath, directory))
if err != nil {
return err
return nil, err
}

// Skip the Terragrunt cache dir entirely
if info.IsDir() && info.Name() == options.TerragruntCacheDir {
return filepath.SkipDir
}
directories = append(directories, matches...)
}

isTerragruntModule, err := containsTerragruntModule(path, info, terragruntOptions)
if err != nil {
return err
}
// TODO: since the terragrunt include directories can be specified as globs, we need to expand
// those globs before walking over them.
Comment on lines +373 to +374
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This TODO is now addressed in the for loop above right? If so, we should remove this.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah yes, forgot to take that out. Good catch.


if isTerragruntModule {
configFiles = append(configFiles, GetDefaultConfigPath(path))
}
for _, directory := range directories {
terragruntOptions.Logger.Logf(logrus.InfoLevel, "walking over directory for terragrunt modules: %s", directory)
err := filepath.Walk(directory, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}

return nil
})
// Skip the Terragrunt cache dir entirely
if info.IsDir() && info.Name() == options.TerragruntCacheDir {
return filepath.SkipDir
}

isTerragruntModule, err := containsTerragruntModule(path, info, terragruntOptions)
if err != nil {
return err
}

if isTerragruntModule {
terragruntOptions.Logger.Logf(logrus.InfoLevel, "found terragrunt module: %s", path)
configFiles = append(configFiles, GetDefaultConfigPath(path))
}

return nil
})

if err != nil {
return configFiles, err
}
}

return configFiles, err
// NOTE: we're sorting the configuration files as I found that very rarely the order returned by zglob wasn't
// always determinate. I don't know if this is a bug in the zglob module or just side-effect of a race in the module.
// @celestialorb, 2021/04/29
sort.Strings(configFiles)
return configFiles, nil
}

// Returns true if the given path with the given FileInfo contains a Terragrunt module and false otherwise. A path
Expand Down
81 changes: 70 additions & 11 deletions config/config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1019,9 +1019,9 @@ func TestFindConfigFilesInPathMultipleConfigs(t *testing.T) {
t.Parallel()

expected := []string{
"../test/fixture-config-files/multiple-configs/terragrunt.hcl",
"../test/fixture-config-files/multiple-configs/subdir-2/subdir/terragrunt.hcl",
"../test/fixture-config-files/multiple-configs/subdir-3/terragrunt.hcl",
"../test/fixture-config-files/multiple-configs/terragrunt.hcl",
}
terragruntOptions, err := options.NewTerragruntOptionsForTest("test")
require.NoError(t, err)
Expand All @@ -1036,9 +1036,9 @@ func TestFindConfigFilesInPathMultipleJsonConfigs(t *testing.T) {
t.Parallel()

expected := []string{
"../test/fixture-config-files/multiple-json-configs/terragrunt.hcl.json",
"../test/fixture-config-files/multiple-json-configs/subdir-2/subdir/terragrunt.hcl.json",
"../test/fixture-config-files/multiple-json-configs/subdir-3/terragrunt.hcl.json",
"../test/fixture-config-files/multiple-json-configs/terragrunt.hcl.json",
}
terragruntOptions, err := options.NewTerragruntOptionsForTest("test")
require.NoError(t, err)
Expand All @@ -1053,16 +1053,55 @@ func TestFindConfigFilesInPathMultipleMixedConfigs(t *testing.T) {
t.Parallel()

expected := []string{
"../test/fixture-config-files/multiple-mixed-configs/subdir-2/subdir/terragrunt.hcl",
"../test/fixture-config-files/multiple-mixed-configs/subdir-3/terragrunt.hcl.json",
"../test/fixture-config-files/multiple-mixed-configs/terragrunt.hcl.json",
}
terragruntOptions, err := options.NewTerragruntOptionsForTest("test")
require.NoError(t, err)

actual, err := FindConfigFilesInPath("../test/fixture-config-files/multiple-mixed-configs", terragruntOptions)

assert.Nil(t, err, "Unexpected error: %v", err)
assert.Equal(t, expected, actual)
}

func TestFindConfigFilesInPathIncludeDirs(t *testing.T) {
t.Parallel()

expected := []string{
"../test/fixture-config-files/multiple-mixed-configs/subdir-2/subdir/terragrunt.hcl",
"../test/fixture-config-files/multiple-mixed-configs/subdir-3/terragrunt.hcl.json",
}

terragruntOptions, err := options.NewTerragruntOptionsForTest("test")
require.NoError(t, err)

terragruntOptions.StrictInclude = true
terragruntOptions.IncludeDirs = []string{"subdir-2", "subdir-3"}

actual, err := FindConfigFilesInPath("../test/fixture-config-files/multiple-mixed-configs", terragruntOptions)
assert.Nil(t, err, "Unexpected error: %v", err)

assert.Equal(t, expected, actual)
}

func TestFindConfigFilesInPathStrictInclude(t *testing.T) {
t.Parallel()

expected := []string{
"../test/fixture-config-files/multiple-mixed-configs/subdir-2/subdir/terragrunt.hcl",
}

terragruntOptions, err := options.NewTerragruntOptionsForTest("test")
require.NoError(t, err)

terragruntOptions.StrictInclude = true
terragruntOptions.IncludeDirs = []string{"subdir-2"}

actual, err := FindConfigFilesInPath("../test/fixture-config-files/multiple-mixed-configs", terragruntOptions)
assert.Nil(t, err, "Unexpected error: %v", err)

assert.Equal(t, expected, actual)
}

Expand All @@ -1085,10 +1124,10 @@ func TestFindConfigFilesIgnoresTerraformDataDir(t *testing.T) {
t.Parallel()

expected := []string{
"../test/fixture-config-files/ignore-terraform-data-dir/terragrunt.hcl",
"../test/fixture-config-files/ignore-terraform-data-dir/.tf_data/modules/mod/terragrunt.hcl",
"../test/fixture-config-files/ignore-terraform-data-dir/subdir/terragrunt.hcl",
"../test/fixture-config-files/ignore-terraform-data-dir/subdir/.tf_data/modules/mod/terragrunt.hcl",
"../test/fixture-config-files/ignore-terraform-data-dir/subdir/terragrunt.hcl",
"../test/fixture-config-files/ignore-terraform-data-dir/terragrunt.hcl",
}
terragruntOptions, err := options.NewTerragruntOptionsForTest("test")
require.NoError(t, err)
Expand All @@ -1103,9 +1142,9 @@ func TestFindConfigFilesIgnoresTerraformDataDirEnv(t *testing.T) {
t.Parallel()

expected := []string{
"../test/fixture-config-files/ignore-terraform-data-dir/terragrunt.hcl",
"../test/fixture-config-files/ignore-terraform-data-dir/subdir/terragrunt.hcl",
"../test/fixture-config-files/ignore-terraform-data-dir/subdir/.terraform/modules/mod/terragrunt.hcl",
"../test/fixture-config-files/ignore-terraform-data-dir/subdir/terragrunt.hcl",
"../test/fixture-config-files/ignore-terraform-data-dir/terragrunt.hcl",
}
terragruntOptions, err := options.NewTerragruntOptionsForTest("test")
require.NoError(t, err)
Expand All @@ -1121,10 +1160,10 @@ func TestFindConfigFilesIgnoresTerraformDataDirEnvPath(t *testing.T) {
t.Parallel()

expected := []string{
"../test/fixture-config-files/ignore-terraform-data-dir/terragrunt.hcl",
"../test/fixture-config-files/ignore-terraform-data-dir/.tf_data/modules/mod/terragrunt.hcl",
"../test/fixture-config-files/ignore-terraform-data-dir/subdir/terragrunt.hcl",
"../test/fixture-config-files/ignore-terraform-data-dir/subdir/.terraform/modules/mod/terragrunt.hcl",
"../test/fixture-config-files/ignore-terraform-data-dir/subdir/terragrunt.hcl",
"../test/fixture-config-files/ignore-terraform-data-dir/terragrunt.hcl",
}
terragruntOptions, err := options.NewTerragruntOptionsForTest("test")
require.NoError(t, err)
Expand All @@ -1143,10 +1182,10 @@ func TestFindConfigFilesIgnoresTerraformDataDirEnvRoot(t *testing.T) {
require.NoError(t, err)

expected := []string{
filepath.Join(cwd, "../test/fixture-config-files/ignore-terraform-data-dir/terragrunt.hcl"),
filepath.Join(cwd, "../test/fixture-config-files/ignore-terraform-data-dir/subdir/terragrunt.hcl"),
filepath.Join(cwd, "../test/fixture-config-files/ignore-terraform-data-dir/subdir/.terraform/modules/mod/terragrunt.hcl"),
filepath.Join(cwd, "../test/fixture-config-files/ignore-terraform-data-dir/subdir/.tf_data/modules/mod/terragrunt.hcl"),
filepath.Join(cwd, "../test/fixture-config-files/ignore-terraform-data-dir/subdir/terragrunt.hcl"),
filepath.Join(cwd, "../test/fixture-config-files/ignore-terraform-data-dir/terragrunt.hcl"),
}
workingDir := filepath.Join(cwd, "../test/fixture-config-files/ignore-terraform-data-dir/")
terragruntOptions, err := options.NewTerragruntOptionsForTest(workingDir)
Expand All @@ -1163,8 +1202,8 @@ func TestFindConfigFilesIgnoresDownloadDir(t *testing.T) {
t.Parallel()

expected := []string{
"../test/fixture-config-files/multiple-configs/terragrunt.hcl",
"../test/fixture-config-files/multiple-configs/subdir-3/terragrunt.hcl",
"../test/fixture-config-files/multiple-configs/terragrunt.hcl",
}
terragruntOptions, err := options.NewTerragruntOptionsForTest("test")
require.NoError(t, err)
Expand All @@ -1176,6 +1215,26 @@ func TestFindConfigFilesIgnoresDownloadDir(t *testing.T) {
assert.Equal(t, expected, actual)
}

func TestFindConfigFilesInPathWithGlob(t *testing.T) {
t.Parallel()

expected := []string{
"../test/fixture-config-files/multiple-mixed-configs/subdir-2/subdir/terragrunt.hcl",
"../test/fixture-config-files/multiple-mixed-configs/subdir-3/terragrunt.hcl.json",
}

terragruntOptions, err := options.NewTerragruntOptionsForTest("test")
require.NoError(t, err)

terragruntOptions.StrictInclude = true
terragruntOptions.IncludeDirs = []string{"subdir-*"}

actual, err := FindConfigFilesInPath("../test/fixture-config-files/multiple-mixed-configs", terragruntOptions)
assert.Nil(t, err, "Unexpected error: %v", err)

assert.Equal(t, expected, actual)
}

func mockOptionsForTestWithConfigPath(t *testing.T, configPath string) *options.TerragruntOptions {
opts, err := options.NewTerragruntOptionsForTest(configPath)
if err != nil {
Expand Down