diff --git a/.circleci/config.yml b/.circleci/config.yml index 556c512e7b..616cc7584d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -761,6 +761,7 @@ workflows: only: /^v.*/ context: - AWS__PHXDEVOPS__circle-ci-test + - AWS__PHXDEVOPS__terragrunt-oidc-test - GCP__automated-tests - GITHUB__PAT__gruntwork-ci - APPLE__OSX__code-signing @@ -770,6 +771,7 @@ workflows: only: /^v.*/ context: - AWS__PHXDEVOPS__circle-ci-test + - AWS__PHXDEVOPS__terragrunt-oidc-test - GCP__automated-tests - GITHUB__PAT__gruntwork-ci - APPLE__OSX__code-signing @@ -779,6 +781,7 @@ workflows: only: /^v.*/ context: - AWS__PHXDEVOPS__circle-ci-test + - AWS__PHXDEVOPS__terragrunt-oidc-test - GCP__automated-tests - GITHUB__PAT__gruntwork-ci - APPLE__OSX__code-signing diff --git a/cli/app.go b/cli/app.go index 11cf909306..abf639173c 100644 --- a/cli/app.go +++ b/cli/app.go @@ -340,6 +340,21 @@ func initialSetup(cliCtx *cli.Context, opts *options.TerragruntOptions) error { opts.ExcludeByDefault = true } + if !opts.ExcludeByDefault && len(opts.ModulesThatInclude) > 0 { + opts.Logger.Debugf("Modules that include set. Excluding by default.") + opts.ExcludeByDefault = true + } + + if !opts.ExcludeByDefault && len(opts.UnitsReading) > 0 { + opts.Logger.Debugf("Units that read set. Excluding by default.") + opts.ExcludeByDefault = true + } + + if !opts.ExcludeByDefault && opts.StrictInclude { + opts.Logger.Debugf("Strict include set. Excluding by default.") + opts.ExcludeByDefault = true + } + opts.IncludeDirs, err = util.GlobCanonicalPath(opts.WorkingDir, opts.IncludeDirs...) if err != nil { return err diff --git a/cli/commands/flags.go b/cli/commands/flags.go index f5dca7ca1e..4d959f264d 100644 --- a/cli/commands/flags.go +++ b/cli/commands/flags.go @@ -62,30 +62,6 @@ const ( TerragruntIAMWebIdentityTokenFlagName = "terragrunt-iam-web-identity-token" TerragruntIAMWebIdentityTokenEnvName = "TERRAGRUNT_IAM_ASSUME_ROLE_WEB_IDENTITY_TOKEN" - TerragruntIgnoreDependencyErrorsFlagName = "terragrunt-ignore-dependency-errors" - TerragruntIgnoreDependencyErrorsEnvName = "TERRAGRUNT_IGNORE_DEPENDENCY_ERRORS" - - TerragruntIgnoreDependencyOrderFlagName = "terragrunt-ignore-dependency-order" - TerragruntIgnoreDependencyOrderEnvName = "TERRAGRUNT_IGNORE_DEPENDENCY_ORDER" - - TerragruntIgnoreExternalDependenciesFlagName = "terragrunt-ignore-external-dependencies" - TerragruntIgnoreExternalDependenciesEnvName = "TERRAGRUNT_IGNORE_EXTERNAL_DEPENDENCIES" - - TerragruntIncludeExternalDependenciesFlagName = "terragrunt-include-external-dependencies" - TerragruntIncludeExternalDependenciesEnvName = "TERRAGRUNT_INCLUDE_EXTERNAL_DEPENDENCIES" - - TerragruntExcludesFileFlagName = "terragrunt-excludes-file" - TerragruntExcludesFileEnvName = "TERRAGRUNT_EXCLUDES_FILE" - - TerragruntExcludeDirFlagName = "terragrunt-exclude-dir" - TerragruntExcludeDirEnvName = "TERRAGRUNT_EXCLUDE_DIR" - - TerragruntIncludeDirFlagName = "terragrunt-include-dir" - TerragruntIncludeDirEnvName = "TERRAGRUNT_INCLUDE_DIR" - - TerragruntStrictIncludeFlagName = "terragrunt-strict-include" - TerragruntStrictIncludeEnvName = "TERRAGRUNT_STRICT_INCLUDE" - TerragruntParallelismFlagName = "terragrunt-parallelism" TerragruntParallelismEnvName = "TERRAGRUNT_PARALLELISM" @@ -125,6 +101,35 @@ const ( TerragruntNoDestroyDependenciesCheckFlagEnvName = "TERRAGRUNT_NO_DESTROY_DEPENDENCIES_CHECK" TerragruntNoDestroyDependenciesCheckFlagName = "terragrunt-no-destroy-dependencies-check" + // Queue related flags + + TerragruntIgnoreDependencyErrorsFlagName = "terragrunt-ignore-dependency-errors" + TerragruntIgnoreDependencyErrorsEnvName = "TERRAGRUNT_IGNORE_DEPENDENCY_ERRORS" + + TerragruntIgnoreDependencyOrderFlagName = "terragrunt-ignore-dependency-order" + TerragruntIgnoreDependencyOrderEnvName = "TERRAGRUNT_IGNORE_DEPENDENCY_ORDER" + + TerragruntIgnoreExternalDependenciesFlagName = "terragrunt-ignore-external-dependencies" + TerragruntIgnoreExternalDependenciesEnvName = "TERRAGRUNT_IGNORE_EXTERNAL_DEPENDENCIES" + + TerragruntIncludeExternalDependenciesFlagName = "terragrunt-include-external-dependencies" + TerragruntIncludeExternalDependenciesEnvName = "TERRAGRUNT_INCLUDE_EXTERNAL_DEPENDENCIES" + + TerragruntExcludesFileFlagName = "terragrunt-excludes-file" + TerragruntExcludesFileEnvName = "TERRAGRUNT_EXCLUDES_FILE" + + TerragruntExcludeDirFlagName = "terragrunt-exclude-dir" + TerragruntExcludeDirEnvName = "TERRAGRUNT_EXCLUDE_DIR" + + TerragruntIncludeDirFlagName = "terragrunt-include-dir" + TerragruntIncludeDirEnvName = "TERRAGRUNT_INCLUDE_DIR" + + TerragruntStrictIncludeFlagName = "terragrunt-strict-include" + TerragruntStrictIncludeEnvName = "TERRAGRUNT_STRICT_INCLUDE" + + TerragruntUnitsReadingFlagName = "terragrunt-queue-include-units-reading" + TerragruntUnitsReadingEnvName = "TERRAGRUNT_QUEUE_INCLUDE_UNITS_READING" + // Logs related flags/envs TerragruntLogLevelFlagName = "terragrunt-log-level" @@ -470,6 +475,12 @@ func NewGlobalFlags(opts *options.TerragruntOptions) cli.Flags { Destination: &opts.ModulesThatInclude, Usage: "If flag is set, 'run-all' will only run the command against Terragrunt modules that include the specified file.", }, + &cli.SliceFlag[string]{ + Name: TerragruntUnitsReadingFlagName, + EnvVar: TerragruntUnitsReadingEnvName, + Destination: &opts.UnitsReading, + Usage: "If flag is set, 'run-all' will only run the command against Terragrunt units that read the specified file via an HCL function.", + }, &cli.BoolFlag{ Name: TerragruntFailOnStateBucketCreationFlagName, EnvVar: TerragruntFailOnStateBucketCreationEnvName, diff --git a/cli/commands/terraform/creds/providers/externalcmd/provider.go b/cli/commands/terraform/creds/providers/externalcmd/provider.go index 90f44192a9..07e894de1a 100644 --- a/cli/commands/terraform/creds/providers/externalcmd/provider.go +++ b/cli/commands/terraform/creds/providers/externalcmd/provider.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/gruntwork-io/terragrunt/cli/commands/terraform/creds/providers" + "github.com/gruntwork-io/terragrunt/cli/commands/terraform/creds/providers/amazonsts" "github.com/gruntwork-io/terragrunt/internal/errors" "github.com/gruntwork-io/terragrunt/options" "github.com/gruntwork-io/terragrunt/shell" @@ -66,10 +67,21 @@ func (provider *Provider) GetCredentials(ctx context.Context) (*providers.Creden } if resp.AWSCredentials != nil { - if envs := resp.AWSCredentials.Envs(provider.terragruntOptions); envs != nil { + if envs := resp.AWSCredentials.Envs(ctx, provider.terragruntOptions); envs != nil { provider.terragruntOptions.Logger.Debugf("Obtaining AWS credentials from the %s.", provider.Name()) maps.Copy(creds.Envs, envs) } + + return creds, nil + } + + if resp.AWSRole != nil { + if envs := resp.AWSRole.Envs(ctx, provider.terragruntOptions); envs != nil { + provider.terragruntOptions.Logger.Debugf("Assuming AWS role %s using the %s.", resp.AWSRole.RoleARN, provider.Name()) + maps.Copy(creds.Envs, envs) + } + + return creds, nil } return creds, nil @@ -77,6 +89,7 @@ func (provider *Provider) GetCredentials(ctx context.Context) (*providers.Creden type Response struct { AWSCredentials *AWSCredentials `json:"awsCredentials"` + AWSRole *AWSRole `json:"awsRole"` Envs map[string]string `json:"envs"` } @@ -86,7 +99,67 @@ type AWSCredentials struct { SessionToken string `json:"SESSION_TOKEN"` } -func (creds *AWSCredentials) Envs(opts *options.TerragruntOptions) map[string]string { +type AWSRole struct { + RoleARN string `json:"roleARN"` + RoleSessionName string `json:"roleSessionName"` + Duration int64 `json:"duration"` + WebIdentityToken string `json:"webIdentityToken"` +} + +func (role *AWSRole) Envs(ctx context.Context, opts *options.TerragruntOptions) map[string]string { + if role.RoleARN == "" { + opts.Logger.Warnf("The command %s completed successfully, but AWS role assumption contains empty required value: roleARN, nothing is being done.", opts.AuthProviderCmd) + return nil + } + + sessionName := role.RoleSessionName + if sessionName == "" { + sessionName = options.GetDefaultIAMAssumeRoleSessionName() + } + + duration := role.Duration + if duration == 0 { + duration = options.DefaultIAMAssumeRoleDuration + } + + // Construct minimal TerragruntOptions for role assumption. + providerOpts := options.TerragruntOptions{ + IAMRoleOptions: options.IAMRoleOptions{ + RoleARN: role.RoleARN, + AssumeRoleDuration: duration, + AssumeRoleSessionName: sessionName, + }, + Logger: opts.Logger, + } + + if role.WebIdentityToken != "" { + providerOpts.IAMRoleOptions.WebIdentityToken = role.WebIdentityToken + } + + provider := amazonsts.NewProvider(&providerOpts) + + creds, err := provider.GetCredentials(ctx) + if err != nil { + opts.Logger.Warnf("Failed to assume role %s: %v", role.RoleARN, err) + return nil + } + + if creds == nil { + opts.Logger.Warnf("The command %s completed successfully, but failed to assume role %s, nothing is being done.", opts.AuthProviderCmd, role.RoleARN) + return nil + } + + envs := map[string]string{ + "AWS_ACCESS_KEY_ID": creds.Envs["AWS_ACCESS_KEY_ID"], + "AWS_SECRET_ACCESS_KEY": creds.Envs["AWS_SECRET_ACCESS_KEY"], + "AWS_SESSION_TOKEN": creds.Envs["AWS_SESSION_TOKEN"], + "AWS_SECURITY_TOKEN": creds.Envs["AWS_SESSION_TOKEN"], + } + + return envs +} + +func (creds *AWSCredentials) Envs(_ context.Context, opts *options.TerragruntOptions) map[string]string { var emptyFields []string if creds.AccessKeyID == "" { diff --git a/config/config_helpers.go b/config/config_helpers.go index 08f0bafc6f..666217b265 100644 --- a/config/config_helpers.go +++ b/config/config_helpers.go @@ -68,6 +68,7 @@ const ( FuncNameEndsWith = "endswith" FuncNameStrContains = "strcontains" FuncNameTimeCmp = "timecmp" + FuncNameMarkAsRead = "mark_as_read" sopsCacheName = "sopsCache" ) @@ -167,6 +168,7 @@ func createTerragruntEvalContext(ctx *ParsingContext, configPath string) (*hcl.E FuncNameGetDefaultRetryableErrors: wrapVoidToStringSliceAsFuncImpl(ctx, getDefaultRetryableErrors), FuncNameReadTfvarsFile: wrapStringSliceToStringAsFuncImpl(ctx, readTFVarsFile), FuncNameGetWorkingDir: wrapVoidToStringAsFuncImpl(ctx, getWorkingDir), + FuncNameMarkAsRead: wrapStringSliceToStringAsFuncImpl(ctx, markAsRead), // Map with HCL functions introduced in Terraform after v0.15.3, since upgrade to a later version is not supported // https://github.com/gruntwork-io/terragrunt/blob/master/go.mod#L22 @@ -627,12 +629,25 @@ func ParseTerragruntConfig(ctx *ParsingContext, configPath string, defaultVal *c targetConfig := getCleanedTargetConfigPath(configPath, ctx.TerragruntOptions.TerragruntConfigPath) targetConfigFileExists := util.FileExists(targetConfig) + if !targetConfigFileExists && defaultVal == nil { return cty.NilVal, errors.New(TerragruntConfigNotFoundError{Path: targetConfig}) - } else if !targetConfigFileExists { + } + + if !targetConfigFileExists { return *defaultVal, nil } + path, err := util.CanonicalPath(targetConfig, ctx.TerragruntOptions.WorkingDir) + if err != nil { + return cty.NilVal, errors.New(err) + } + + ctx.TerragruntOptions.AppendReadFile( + path, + ctx.TerragruntOptions.WorkingDir, + ) + // We update the ctx of terragruntOptions to the config being read in. opts, err := ctx.TerragruntOptions.Clone(targetConfig) if err != nil { @@ -812,6 +827,11 @@ func sopsDecryptFile(ctx *ParsingContext, params []string) (string, error) { return "", errors.New(err) } + ctx.TerragruntOptions.AppendReadFile( + canonicalSourceFile, + ctx.TerragruntOptions.WorkingDir, + ) + // Set environment variables from the TerragruntOptions.Env map. // This is especially useful for integrations with things like the `terragrunt-auth-provider` flag, // which can set environment variables that are used for decryption. @@ -1008,6 +1028,11 @@ func readTFVarsFile(ctx *ParsingContext, args []string) (string, error) { return "", errors.New(TFVarFileNotFoundError{File: varFile}) } + ctx.TerragruntOptions.AppendReadFile( + varFile, + ctx.TerragruntOptions.WorkingDir, + ) + fileContents, err := os.ReadFile(varFile) if err != nil { return "", errors.New(fmt.Errorf("could not read file %q: %w", varFile, err)) @@ -1036,6 +1061,33 @@ func readTFVarsFile(ctx *ParsingContext, args []string) (string, error) { return string(data), nil } +// markAsRead marks a file as explicitly read. This is useful for detection via TerragruntUnitsReading flag. +func markAsRead(ctx *ParsingContext, args []string) (string, error) { + if len(args) != 1 { + return "", errors.New(WrongNumberOfParamsError{Func: "mark_as_read", Expected: "1", Actual: len(args)}) + } + + file := args[0] + + path, err := util.CanonicalPath(file, ctx.TerragruntOptions.WorkingDir) + if err != nil { + return "", errors.New(err) + } + + ctx.TerragruntOptions.AppendReadFile( + path, + ctx.TerragruntOptions.WorkingDir, + ) + + return file, nil +} + +// warnWhenFileNotMarkedAsRead warns when a file is not being marked as read, even though a user might expect it to be. +// Situations where this is the case include: +// - A user specifies a file in the UnitsReading flag and that file is being read while parsing the inputs attribute. +// +// When the file is not marked as read, the function will return true, otherwise false. + // ParseAndDecodeVarFile uses the HCL2 file to parse the given varfile string into an HCL file body, and then decode it // into the provided output. func ParseAndDecodeVarFile(opts *options.TerragruntOptions, varFile string, fileContents []byte, out interface{}) error { diff --git a/configstack/module.go b/configstack/module.go index 2bf4f7176a..ceb1e4bece 100644 --- a/configstack/module.go +++ b/configstack/module.go @@ -96,11 +96,11 @@ func (module *TerraformModule) checkForCyclesUsingDepthFirstSearch(visitedPaths } // planFile - return plan file location, if output folder is set -func (module *TerraformModule) planFile(terragruntOptions *options.TerragruntOptions) string { +func (module *TerraformModule) planFile(opts *options.TerragruntOptions) string { var planFile string // set plan file location if output folder is set - planFile = module.outputFile(terragruntOptions) + planFile = module.outputFile(opts) planCommand := module.TerragruntOptions.TerraformCommand == terraform.CommandNamePlan || module.TerragruntOptions.TerraformCommand == terraform.CommandNameShow @@ -158,26 +158,26 @@ func (module *TerraformModule) findModuleInPath(targetDirs []string) bool { // Note that we skip the prompt for `run-all destroy` calls. Given the destructive and irreversible nature of destroy, we don't // want to provide any risk to the user of accidentally destroying an external dependency unless explicitly included // with the --terragrunt-include-external-dependencies or --terragrunt-include-dir flags. -func (module *TerraformModule) confirmShouldApplyExternalDependency(ctx context.Context, dependency *TerraformModule, terragruntOptions *options.TerragruntOptions) (bool, error) { - if terragruntOptions.IncludeExternalDependencies { - terragruntOptions.Logger.Debugf("The --terragrunt-include-external-dependencies flag is set, so automatically including all external dependencies, and will run this command against module %s, which is a dependency of module %s.", dependency.Path, module.Path) +func (module *TerraformModule) confirmShouldApplyExternalDependency(ctx context.Context, dependency *TerraformModule, opts *options.TerragruntOptions) (bool, error) { + if opts.IncludeExternalDependencies { + opts.Logger.Debugf("The --terragrunt-include-external-dependencies flag is set, so automatically including all external dependencies, and will run this command against module %s, which is a dependency of module %s.", dependency.Path, module.Path) return true, nil } - if terragruntOptions.NonInteractive { - terragruntOptions.Logger.Debugf("The --non-interactive flag is set. To avoid accidentally affecting external dependencies with a run-all command, will not run this command against module %s, which is a dependency of module %s.", dependency.Path, module.Path) + if opts.NonInteractive { + opts.Logger.Debugf("The --non-interactive flag is set. To avoid accidentally affecting external dependencies with a run-all command, will not run this command against module %s, which is a dependency of module %s.", dependency.Path, module.Path) return false, nil } - stackCmd := terragruntOptions.TerraformCommand + stackCmd := opts.TerraformCommand if stackCmd == "destroy" { - terragruntOptions.Logger.Debugf("run-all command called with destroy. To avoid accidentally having destructive effects on external dependencies with run-all command, will not run this command against module %s, which is a dependency of module %s.", dependency.Path, module.Path) + opts.Logger.Debugf("run-all command called with destroy. To avoid accidentally having destructive effects on external dependencies with run-all command, will not run this command against module %s, which is a dependency of module %s.", dependency.Path, module.Path) return false, nil } - terragruntOptions.Logger.Infof("Module %s has external dependency %s", module.Path, dependency.Path) + opts.Logger.Infof("Module %s has external dependency %s", module.Path, dependency.Path) - return shell.PromptUserForYesNo(ctx, "Should Terragrunt apply the external dependency?", terragruntOptions) + return shell.PromptUserForYesNo(ctx, "Should Terragrunt apply the external dependency?", opts) } // Get the list of modules this module depends on @@ -220,15 +220,15 @@ type TerraformModules []*TerraformModule // FindWhereWorkingDirIsIncluded - find where working directory is included, flow: // 1. Find root git top level directory and build list of modules -// 2. Iterate over includes from terragruntOptions if git top level directory detection failed +// 2. Iterate over includes from opts if git top level directory detection failed // 3. Filter found module only items which has in dependencies working directory -func FindWhereWorkingDirIsIncluded(ctx context.Context, terragruntOptions *options.TerragruntOptions, terragruntConfig *config.TerragruntConfig) TerraformModules { +func FindWhereWorkingDirIsIncluded(ctx context.Context, opts *options.TerragruntOptions, terragruntConfig *config.TerragruntConfig) TerraformModules { var ( pathsToCheck []string matchedModulesMap = make(TerraformModulesMap) ) - if gitTopLevelDir, err := shell.GitTopLevelDir(ctx, terragruntOptions, terragruntOptions.WorkingDir); err == nil { + if gitTopLevelDir, err := shell.GitTopLevelDir(ctx, opts, opts.WorkingDir); err == nil { pathsToCheck = append(pathsToCheck, gitTopLevelDir) } else { // detection failed, trying to use include directories as source for stacks @@ -247,14 +247,14 @@ func FindWhereWorkingDirIsIncluded(ctx context.Context, terragruntOptions *optio cfgOptions, err := options.NewTerragruntOptionsWithConfigPath(dir) if err != nil { - terragruntOptions.Logger.Debugf("Failed to build terragrunt options from %s %v", dir, err) + opts.Logger.Debugf("Failed to build terragrunt options from %s %v", dir, err) continue } - cfgOptions.Env = terragruntOptions.Env - cfgOptions.LogLevel = terragruntOptions.LogLevel - cfgOptions.OriginalTerragruntConfigPath = terragruntOptions.OriginalTerragruntConfigPath - cfgOptions.TerraformCommand = terragruntOptions.TerraformCommand + cfgOptions.Env = opts.Env + cfgOptions.LogLevel = opts.LogLevel + cfgOptions.OriginalTerragruntConfigPath = opts.OriginalTerragruntConfigPath + cfgOptions.TerraformCommand = opts.TerraformCommand cfgOptions.NonInteractive = true cfgOptions.Logger.SetOptions(log.WithHooks(NewForceLogLevelHook(log.DebugLevel))) @@ -263,13 +263,13 @@ func FindWhereWorkingDirIsIncluded(ctx context.Context, terragruntOptions *optio if err != nil { // log error as debug since in some cases stack building may fail because parent files can be designed // to work with relative paths from downstream modules - terragruntOptions.Logger.Debugf("Failed to build module stack %v", err) + opts.Logger.Debugf("Failed to build module stack %v", err) continue } dependentModules := stack.ListStackDependentModules() - deps, found := dependentModules[terragruntOptions.WorkingDir] + deps, found := dependentModules[opts.WorkingDir] if found { for _, module := range stack.Modules { for _, dep := range deps { @@ -295,19 +295,19 @@ func FindWhereWorkingDirIsIncluded(ctx context.Context, terragruntOptions *optio // for a directed graph. It can be used to dump a .dot file. // This is a similar implementation to terraform's digraph https://github.com/hashicorp/terraform/blob/master/digraph/graphviz.go // adding some styling to modules that are excluded from the execution in *-all commands -func (modules TerraformModules) WriteDot(w io.Writer, terragruntOptions *options.TerragruntOptions) error { +func (modules TerraformModules) WriteDot(w io.Writer, opts *options.TerragruntOptions) error { if _, err := w.Write([]byte("digraph {\n")); err != nil { return errors.New(err) } defer func(w io.Writer, p []byte) { _, err := w.Write(p) if err != nil { - terragruntOptions.Logger.Warnf("Failed to close graphviz output: %v", err) + opts.Logger.Warnf("Failed to close graphviz output: %v", err) } }(w, []byte("}\n")) // all paths are relative to the TerragruntConfigPath - prefix := filepath.Dir(terragruntOptions.TerragruntConfigPath) + "/" + prefix := filepath.Dir(opts.TerragruntConfigPath) + "/" for _, source := range modules { // apply a different coloring for excluded nodes @@ -407,41 +407,17 @@ func (modules TerraformModules) CheckForCycles() error { return nil } -// flagExcludedDirs iterates over a module slice and flags all entries as excluded, which should be ignored via the terragrunt-exclude-dir CLI flag. -func (modules TerraformModules) flagExcludedDirs(terragruntOptions *options.TerragruntOptions) TerraformModules { - for _, module := range modules { - if module.findModuleInPath(terragruntOptions.ExcludeDirs) { - // Mark module itself as excluded - module.FlagExcluded = true - } - - // Mark all affected dependencies as excluded - for _, dependency := range module.Dependencies { - if dependency.findModuleInPath(terragruntOptions.ExcludeDirs) { - dependency.FlagExcluded = true - } - } - } - - return modules -} - -// flagIncludedDirs iterates over a module slice and flags all entries not in the list specified via the terragrunt-include-dir CLI flag as excluded. -func (modules TerraformModules) flagIncludedDirs(terragruntOptions *options.TerragruntOptions) TerraformModules { - // If we're not excluding by default, we should include everything by default. - // This can happen when a user doesn't set include flags. - if !terragruntOptions.ExcludeByDefault { - // If we aren't given any include directories, but are given the strict include flag, - // return no modules. - if terragruntOptions.StrictInclude { - return TerraformModules{} - } - +// flagIncludedDirs includes all units by default. +// +// However, when anything that triggers ExcludeByDefault is set, the function will instead +// selectively include only the units that are in the list specified via the IncludeDirs option. +func (modules TerraformModules) flagIncludedDirs(opts *options.TerragruntOptions) TerraformModules { + if !opts.ExcludeByDefault { return modules } for _, module := range modules { - if module.findModuleInPath(terragruntOptions.IncludeDirs) { + if module.findModuleInPath(opts.IncludeDirs) { module.FlagExcluded = false } else { module.FlagExcluded = true @@ -449,7 +425,7 @@ func (modules TerraformModules) flagIncludedDirs(terragruntOptions *options.Terr } // Mark all affected dependencies as included before proceeding if not in strict include mode. - if !terragruntOptions.StrictInclude { + if !opts.StrictInclude { for _, module := range modules { if !module.FlagExcluded { for _, dependency := range module.Dependencies { @@ -462,36 +438,30 @@ func (modules TerraformModules) flagIncludedDirs(terragruntOptions *options.Terr return modules } -// flagModulesThatDontInclude iterates over a module slice and flags all modules that don't include at least one file in -// the specified include list on the TerragruntOptions ModulesThatInclude attribute. Flagged modules will be filtered -// out of the set. -func (modules TerraformModules) flagModulesThatDontInclude(terragruntOptions *options.TerragruntOptions) (TerraformModules, error) { - // If no ModulesThatInclude is specified return the modules list instantly - if len(terragruntOptions.ModulesThatInclude) == 0 { +// flagUnitsThatAreIncluded iterates over a module slice and flags all modules that include at least one file in +// the specified include list on the TerragruntOptions ModulesThatInclude attribute. +func (modules TerraformModules) flagUnitsThatAreIncluded(opts *options.TerragruntOptions) (TerraformModules, error) { + // The two flags ModulesThatInclude and UnitsReading should both be considered when determining which + // units to include in the run queue. + unitsThatInclude := append(opts.ModulesThatInclude, opts.UnitsReading...) + + // If no unitsThatInclude is specified return the modules list instantly + if len(unitsThatInclude) == 0 { return modules, nil } - modulesThatIncludeCanonicalPath := []string{} + modulesThatIncludeCanonicalPaths := []string{} - for _, includePath := range terragruntOptions.ModulesThatInclude { - canonicalPath, err := util.CanonicalPath(includePath, terragruntOptions.WorkingDir) + for _, includePath := range unitsThatInclude { + canonicalPath, err := util.CanonicalPath(includePath, opts.WorkingDir) if err != nil { return nil, err } - modulesThatIncludeCanonicalPath = append(modulesThatIncludeCanonicalPath, canonicalPath) + modulesThatIncludeCanonicalPaths = append(modulesThatIncludeCanonicalPaths, canonicalPath) } for _, module := range modules { - // Ignore modules that are already excluded because this feature is a filter for excluding the subset, not - // including modules that have already been excluded through other means. - if module.FlagExcluded { - continue - } - - // Mark modules that don't include any of the specified paths as excluded. To do this, we first flag the module - // as excluded, and if it includes any path in the set, we set the exclude flag back to false. - module.FlagExcluded = true for _, includeConfig := range module.Config.ProcessedIncludes { // resolve include config to canonical path to compare with modulesThatIncludeCanonicalPath // https://github.com/gruntwork-io/terragrunt/issues/1944 @@ -500,7 +470,7 @@ func (modules TerraformModules) flagModulesThatDontInclude(terragruntOptions *op return nil, err } - if util.ListContainsElement(modulesThatIncludeCanonicalPath, canonicalPath) { + if util.ListContainsElement(modulesThatIncludeCanonicalPaths, canonicalPath) { module.FlagExcluded = false } } @@ -512,14 +482,13 @@ func (modules TerraformModules) flagModulesThatDontInclude(terragruntOptions *op continue } - dependency.FlagExcluded = true for _, includeConfig := range dependency.Config.ProcessedIncludes { canonicalPath, err := util.CanonicalPath(includeConfig.Path, module.Path) if err != nil { return nil, err } - if util.ListContainsElement(modulesThatIncludeCanonicalPath, canonicalPath) { + if util.ListContainsElement(modulesThatIncludeCanonicalPaths, canonicalPath) { dependency.FlagExcluded = false } } @@ -529,6 +498,54 @@ func (modules TerraformModules) flagModulesThatDontInclude(terragruntOptions *op return modules, nil } +// flagUnitsThatRead iterates over a module slice and flags all modules that read at least one file in the specified +// file list in the TerragruntOptions UnitsReading attribute. +func (modules TerraformModules) flagUnitsThatRead(opts *options.TerragruntOptions) (TerraformModules, error) { + // If no UnitsThatRead is specified return the modules list instantly + if len(opts.UnitsReading) == 0 { + return modules, nil + } + + for _, readPath := range opts.UnitsReading { + path, err := util.CanonicalPath(readPath, opts.WorkingDir) + if err != nil { + return nil, err + } + + for _, module := range modules { + if opts.DidReadFile(path, module.Path) { + module.FlagExcluded = false + } + } + } + + return modules, nil +} + +// flagExcludedDirs iterates over a module slice and flags all entries as excluded listed in the terragrunt-exclude-dir CLI flag. +func (modules TerraformModules) flagExcludedDirs(opts *options.TerragruntOptions) TerraformModules { + // If we don't have any excludes, we don't need to do anything. + if len(opts.ExcludeDirs) == 0 { + return modules + } + + for _, module := range modules { + if module.findModuleInPath(opts.ExcludeDirs) { + // Mark module itself as excluded + module.FlagExcluded = true + } + + // Mark all affected dependencies as excluded + for _, dependency := range module.Dependencies { + if dependency.findModuleInPath(opts.ExcludeDirs) { + dependency.FlagExcluded = true + } + } + } + + return modules +} + var existingModules = cache.NewCache[*TerraformModulesMap](existingModulesCacheName) type TerraformModulesMap map[string]*TerraformModule diff --git a/configstack/stack.go b/configstack/stack.go index b037e4ea93..0f99a72e2f 100644 --- a/configstack/stack.go +++ b/configstack/stack.go @@ -399,12 +399,12 @@ func (stack *Stack) ResolveTerraformModules(ctx context.Context, terragruntConfi return nil, err } - var includedModules TerraformModules + var withUnitsIncluded TerraformModules err = telemetry.Telemetry(ctx, stack.terragruntOptions, "flag_included_dirs", map[string]interface{}{ "working_dir": stack.terragruntOptions.WorkingDir, }, func(childCtx context.Context) error { - includedModules = crossLinkedModules.flagIncludedDirs(stack.terragruntOptions) + withUnitsIncluded = crossLinkedModules.flagIncludedDirs(stack.terragruntOptions) return nil }) @@ -412,12 +412,18 @@ func (stack *Stack) ResolveTerraformModules(ctx context.Context, terragruntConfi return nil, err } - var includedModulesWithExcluded TerraformModules + var withUnitsThatAreIncludedByOthers TerraformModules - err = telemetry.Telemetry(ctx, stack.terragruntOptions, "flag_excluded_dirs", map[string]interface{}{ + err = telemetry.Telemetry(ctx, stack.terragruntOptions, "flag_units_that_are_included", map[string]interface{}{ "working_dir": stack.terragruntOptions.WorkingDir, }, func(childCtx context.Context) error { - includedModulesWithExcluded = includedModules.flagExcludedDirs(stack.terragruntOptions) + result, err := withUnitsIncluded.flagUnitsThatAreIncluded(stack.terragruntOptions) + if err != nil { + return err + } + + withUnitsThatAreIncludedByOthers = result + return nil }) @@ -425,25 +431,39 @@ func (stack *Stack) ResolveTerraformModules(ctx context.Context, terragruntConfi return nil, err } - var finalModules TerraformModules + var withUnitsRead TerraformModules - err = telemetry.Telemetry(ctx, stack.terragruntOptions, "flag_modules_that_dont_include", map[string]interface{}{ + err = telemetry.Telemetry(ctx, stack.terragruntOptions, "flag_units_that_read", map[string]interface{}{ "working_dir": stack.terragruntOptions.WorkingDir, }, func(childCtx context.Context) error { - result, err := includedModulesWithExcluded.flagModulesThatDontInclude(stack.terragruntOptions) + result, err := withUnitsThatAreIncludedByOthers.flagUnitsThatRead(stack.terragruntOptions) if err != nil { return err } - finalModules = result + withUnitsRead = result return nil }) + if err != nil { return nil, err } - return finalModules, nil + var withModulesExcluded TerraformModules + + err = telemetry.Telemetry(ctx, stack.terragruntOptions, "flag_excluded_dirs", map[string]interface{}{ + "working_dir": stack.terragruntOptions.WorkingDir, + }, func(childCtx context.Context) error { + withModulesExcluded = withUnitsRead.flagExcludedDirs(stack.terragruntOptions) + return nil + }) + + if err != nil { + return nil, err + } + + return withModulesExcluded, nil } // Go through each of the given Terragrunt configuration files and resolve the module that configuration file represents @@ -584,6 +604,9 @@ func (stack *Stack) resolveTerraformModule(ctx context.Context, terragruntConfig }) } + // Hack to persist readFiles. Need to discuss with team to see if there is a better way to handle this. + stack.terragruntOptions.CloneReadFiles(opts.ReadFiles) + terragruntSource, err := config.GetTerragruntSourceForModule(stack.terragruntOptions.Source, modulePath, terragruntConfig) if err != nil { return nil, err diff --git a/docs/_docs/04_reference/built-in-functions.md b/docs/_docs/04_reference/built-in-functions.md index c829cbfe5a..6d09017fb5 100644 --- a/docs/_docs/04_reference/built-in-functions.md +++ b/docs/_docs/04_reference/built-in-functions.md @@ -40,6 +40,7 @@ Terragrunt allows you to use built-in functions anywhere in `terragrunt.hcl`, ju - [sops\_decrypt\_file](#sops_decrypt_file) - [get\_terragrunt\_source\_cli\_flag](#get_terragrunt_source_cli_flag) - [read\_tfvars\_file](#read_tfvars_file) +- [mark\_as\_read](#mark_as_read) ## OpenTofu/Terraform built-in functions @@ -867,3 +868,30 @@ remote_state { } } ``` + +## mark_as_read + +`mark_as_read(file_path)` marks a file as read, so that it can be picked up for inclusion by the [queue-include-units-reading](./cli-options.md#queue-include-units-reading) flag. + +This is useful for situations when you want to mark a file as read, but are not reading it using a native Terragrunt HCL function. + +For example: + +```hcl +locals { + filename = mark_as_read("file-read-by-tofu.txt") +} + +inputs = { + filename = local.filename +} +``` + +By using `mark_as_read` on `file-read-by-tofu.txt`, you can ensure that the `terragrunt.hcl` file passing in the `file-read-by-tofu.txt` file as an input will be included in +any `run-all` run where the flag `--queue-include-units-reading file-read-by-tofu.txt` is set. + +The same technique can be used to mark a file as read when reading a file using code in `run_cmd`, etc. + +**NOTE**: Due to the way that Terragrunt parses configurations during a `run-all`, functions will only properly mark files as read +if they are used in the `locals` block. Reading a file directly in the `inputs` block will not mark the file as read, as the `inputs` +block is not evaluated until *after* the queue has been populated with units to run. diff --git a/docs/_docs/04_reference/cli-options.md b/docs/_docs/04_reference/cli-options.md index 5cb6a68b94..82d9ea7a8c 100644 --- a/docs/_docs/04_reference/cli-options.md +++ b/docs/_docs/04_reference/cli-options.md @@ -48,6 +48,7 @@ This page documents the CLI commands and options available with Terragrunt: - [terragrunt-iam-role](#terragrunt-iam-role) - [terragrunt-iam-assume-role-duration](#terragrunt-iam-assume-role-duration) - [terragrunt-iam-assume-role-session-name](#terragrunt-iam-assume-role-session-name) + - [terragrunt-iam-web-identity-token](#terragrunt-iam-web-identity-token) - [terragrunt-excludes-file](#terragrunt-excludes-file) - [terragrunt-exclude-dir](#terragrunt-exclude-dir) - [terragrunt-include-dir](#terragrunt-include-dir) @@ -988,6 +989,14 @@ Uses the specified duration as the session duration (in seconds) for the STS ses Used as the session name for the STS session which assumes the role defined in `--terragrunt-iam-role`. +### terragrunt-iam-web-identity-token + +**CLI Arg**: `--terragrunt-iam-web-identity-token`
+**Environment Variable**: `TERRAGRUNT_IAM_WEB_IDENTITY_TOKEN`
+**Requires an argument**: `--terragrunt-iam-web-identity-token [/path/to/web-identity-token | web-identity-token-value]`
+ +Used as the web identity token for assuming a role temporarily using the AWS Security Token Service (STS) with the [AssumeRoleWithWebIdentity](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html) API. + ### terragrunt-excludes-file **CLI Arg**: `--terragrunt-excludes-file`
@@ -1314,9 +1323,79 @@ and then will apply this filter on the set of modules that it found. You can pass this argument in multiple times to provide a list of include files to consider. When multiple files are passed in, the set will be the union of modules that includes at least one of the files in the list. -NOTE: When using relative paths, the paths are relative to the working directory. This is either the current working +**NOTE**: When using relative paths, the paths are relative to the working directory. This is either the current working directory, or any path passed in to [terragrunt-working-dir](#terragrunt-working-dir). +**TIP**: This flag is functionally covered by the `--terragrunt-queue-include-units-reading` flag, but is more explicitly +only for the `include` configuration block. + +### terragrunt-queue-include-units-reading + +**CLI Arg**: `--terragrunt-queue-include-units-reading`
+**Environment Variable**: `TERRAGRUNT_QUEUE_INCLUDE_UNITS_READING`
+**Commands**: + +- [run-all](#run-all) +- [plan-all (DEPRECATED: use run-all)](#plan-all-deprecated-use-run-all) +- [apply-all (DEPRECATED: use run-all)](#apply-all-deprecated-use-run-all) +- [output-all (DEPRECATED: use run-all)](#output-all-deprecated-use-run-all) +- [destroy-all (DEPRECATED: use run-all)](#destroy-all-deprecated-use-run-all) +- [validate-all (DEPRECATED: use run-all)](#validate-all-deprecated-use-run-all) + +This flag works very similarly to the `--terragrunt-modules-that-include` flag, but instead of looking only for included configurations, +it also looks for configurations that read a given file. + +When passed in, the `*-all` commands will include all units (modules) that read a given file into the queue. This is useful +when you want to trigger an update on all units that read or include a given file using HCL functions in their configurations. + +Consider the following folder structure: + +```tree +. +├── reading-shared-hcl +│   └── terragrunt.hcl +├── also-reading-shared-hcl +│   └── terragrunt.hcl +├── not-reading-shared-hcl +│   └── terragrunt.hcl +└── shared.hcl +``` + +Suppose that `reading-shared-hcl` and `also-reading-shared-hcl` both read `shared.hcl` in their configurations, like so: + +```hcl +locals { + shared = read_terragrunt_config(find_in_parent_folders("shared.hcl")) +} +``` + +If you run the command `run-all init --terragrunt-queue-include-units-reading shared.hcl` from the root folder, both +`reading-shared-hcl` and `also-reading-shared-hcl` will be run; not `not-reading-shared-hcl`. + +This is because the `read_terragrunt_config` HCL function has a special hook that allows Terragrunt to track that it has +read the file `shared.hcl`. This hook is used by all native HCL functions that Terragrunt supports which read files. + +Note, however, that there are certain scenarios where Terragrunt may not be able to track that a file has been read this way. + +For example, you may be using a bash script to read a file via `run_cmd`, or reading the file via OpenTofu code. To support these +use-cases, the [mark_as_read](./built-in-functions.md#mark_as_read) function can be used to manually mark a file as read. + +That would look something like this: + +```hcl +locals { + filename = mark_as_read("file-read-by-tofu.txt") +} + +inputs = { + filename = local.filename +} +``` + +**⚠️**: Due to the way that Terragrunt parses configurations during a `run-all`, functions will only properly mark files as read +if they are used in the `locals` block. Reading a file directly in the `inputs` block will not mark the file as read, as the `inputs` +block is not evaluated until _after_ the queue has been populated with units to run. + ### terragrunt-fetch-dependency-output-from-state **CLI Arg**: `--terragrunt-fetch-dependency-output-from-state`
@@ -1490,6 +1569,12 @@ The output must be valid JSON of the following schema: "SECRET_ACCESS_KEY": "", "SESSION_TOKEN": "" }, + "awsRole": { + "roleARN": "", + "sessionName": "", + "duration": 0, + "webIdentityToken": "" + }, "envs": { "ANY_KEY": "" } @@ -1514,8 +1599,12 @@ Note that more specific configurations (e.g. `awsCredentials`) take precedence o If you would like to set credentials for AWS with this method, you are encouraged to use `awsCredentials` instead of `envs`, as these keys will be validated to conform to the officially supported environment variables expected by the AWS SDK. +Similarly, if you would like Terragrunt to assume an AWS role on your behalf, you are encouraged to use the `awsRole` configuration instead of `envs`. + Other credential configurations will be supported in the future, but until then, if your provider authenticates via environment variables, you can use the `envs` field to fetch credentials dynamically from a secret store, etc before Terragrunt executes any IAC. +**Note**: The `awsRole` configuration is only used when the `awsCredentials` configuration is not present. If both are present, the `awsCredentials` configuration will take precedence. + ### terragrunt-disable-log-formatting DEPRECATED: Use [terragrunt-log-format](#terragrunt-log-format). diff --git a/options/options.go b/options/options.go index a8c5dcef09..eaedc3833e 100644 --- a/options/options.go +++ b/options/options.go @@ -249,6 +249,10 @@ type TerragruntOptions struct { // in this list. ModulesThatInclude []string + // When used with `run-all`, restrict the units in the stack to only those that read at least one of the files + // in this list. + UnitsReading []string + // A command that can be used to run Terragrunt with the given options. This is useful for running Terragrunt // multiple times (e.g. when spinning up a stack of Terraform modules). The actual command is normally defined // in the cli package, which depends on almost all other packages, so we declare it here so that other @@ -357,6 +361,10 @@ type TerragruntOptions struct { // FeatureFlags is a map of feature flags to enable. FeatureFlags map[string]string + + // ReadFiles is a map of files to the Units + // that read them using HCL functions in the unit. + ReadFiles map[string][]string } // TerragruntOptionsFunc is a functional option type used to pass options in certain integration tests @@ -591,6 +599,8 @@ func (opts *TerragruntOptions) Clone(terragruntConfigPath string) (*TerragruntOp IncludeDirs: opts.IncludeDirs, ExcludeByDefault: opts.ExcludeByDefault, ModulesThatInclude: opts.ModulesThatInclude, + UnitsReading: opts.UnitsReading, + ReadFiles: opts.ReadFiles, Parallelism: opts.Parallelism, StrictInclude: opts.StrictInclude, RunTerragrunt: opts.RunTerragrunt, @@ -725,6 +735,50 @@ func (opts *TerragruntOptions) DataDir() string { return util.JoinPath(opts.WorkingDir, tfDataDir) } +// AppendReadFile appends to the list of files read by a given unit. +func (opts *TerragruntOptions) AppendReadFile(file, unit string) { + if opts.ReadFiles == nil { + opts.ReadFiles = map[string][]string{} + } + + for _, u := range opts.ReadFiles[file] { + if u == unit { + return + } + } + + opts.Logger.Debugf("Tracking that file %s was read by %s.", file, unit) + opts.ReadFiles[file] = append(opts.ReadFiles[file], unit) +} + +// DidReadFile checks if a given file was read by a given unit. +func (opts *TerragruntOptions) DidReadFile(file, unit string) bool { + if opts.ReadFiles == nil { + return false + } + + for _, u := range opts.ReadFiles[file] { + if u == unit { + return true + } + } + + return false +} + +// CloneReadFiles creates a copy of the ReadFiles map. +func (opts *TerragruntOptions) CloneReadFiles(readFiles map[string][]string) { + if readFiles == nil { + return + } + + for file, units := range readFiles { + for _, unit := range units { + opts.AppendReadFile(file, unit) + } + } +} + // identifyDefaultWrappedExecutable returns default path used for wrapped executable. func identifyDefaultWrappedExecutable() string { if util.IsCommandExecutable(TofuDefaultPath, "-version") { diff --git a/test/fixtures/assume-role-web-identity/env-var/main.tf b/test/fixtures/assume-role-web-identity/env-var/main.tf deleted file mode 100644 index ec64be1fd6..0000000000 --- a/test/fixtures/assume-role-web-identity/env-var/main.tf +++ /dev/null @@ -1,4 +0,0 @@ -resource "local_file" "test_file" { - content = "test_file" - filename = "${path.module}/test_file.txt" -} diff --git a/test/fixtures/assume-role-web-identity/env-var/terragrunt.hcl b/test/fixtures/assume-role-web-identity/env-var/terragrunt.hcl deleted file mode 100644 index b64cb5d5b8..0000000000 --- a/test/fixtures/assume-role-web-identity/env-var/terragrunt.hcl +++ /dev/null @@ -1,16 +0,0 @@ -iam_role = "__FILL_IN_ASSUME_ROLE__" -iam_web_identity_token = get_env("__FILL_IN_IDENTITY_TOKEN_ENV_VAR__", "") - -remote_state { - backend = "s3" - generate = { - path = "backend.tf" - if_exists = "overwrite_terragrunt" - } - config = { - bucket = "__FILL_IN_BUCKET_NAME__" - key = "${path_relative_to_include()}/terraform.tfstate" - region = "__FILL_IN_REGION__" - encrypt = true - } -} diff --git a/test/fixtures/auth-provider-cmd/oidc/main.tf b/test/fixtures/auth-provider-cmd/oidc/main.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/fixtures/auth-provider-cmd/oidc/mock-auth-cmd.sh b/test/fixtures/auth-provider-cmd/oidc/mock-auth-cmd.sh new file mode 100755 index 0000000000..408b484198 --- /dev/null +++ b/test/fixtures/auth-provider-cmd/oidc/mock-auth-cmd.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -o pipefail + +: "${AWS_TEST_OIDC_ROLE_ARN:?The AWS_TEST_OIDC_ROLE_ARN environment variable must be set.}" +: "${CIRCLE_OIDC_TOKEN_V2:?The CIRCLE_OIDC_TOKEN_V2 environment variable must be set.}" + +jq -n \ + --arg role "$AWS_TEST_OIDC_ROLE_ARN" \ + --arg token "$CIRCLE_OIDC_TOKEN_V2" \ + '{awsRole: {roleARN: $role, webIdentityToken: $token}}' diff --git a/test/fixtures/auth-provider-cmd/oidc/terragrunt.hcl b/test/fixtures/auth-provider-cmd/oidc/terragrunt.hcl new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/fixtures/units-reading/including/terragrunt.hcl b/test/fixtures/units-reading/including/terragrunt.hcl new file mode 100644 index 0000000000..839b3f755e --- /dev/null +++ b/test/fixtures/units-reading/including/terragrunt.hcl @@ -0,0 +1,3 @@ +include "shared" { + path = find_in_parent_folders("shared.hcl") +} diff --git a/test/fixtures/units-reading/reading-from-tf/main.tf b/test/fixtures/units-reading/reading-from-tf/main.tf new file mode 100644 index 0000000000..7ee83502d5 --- /dev/null +++ b/test/fixtures/units-reading/reading-from-tf/main.tf @@ -0,0 +1,4 @@ +variable "filename" {} +output "shared" { + value = jsondecode(file(var.filename)) +} diff --git a/test/fixtures/units-reading/reading-from-tf/terragrunt.hcl b/test/fixtures/units-reading/reading-from-tf/terragrunt.hcl new file mode 100644 index 0000000000..abd30a9359 --- /dev/null +++ b/test/fixtures/units-reading/reading-from-tf/terragrunt.hcl @@ -0,0 +1,7 @@ +locals { + filename = mark_as_read(find_in_parent_folders("shared.json")) +} + +inputs = { + filename = local.filename +} diff --git a/test/fixtures/units-reading/reading-hcl-and-tfvars/main.tf b/test/fixtures/units-reading/reading-hcl-and-tfvars/main.tf new file mode 100644 index 0000000000..674790dc79 --- /dev/null +++ b/test/fixtures/units-reading/reading-hcl-and-tfvars/main.tf @@ -0,0 +1,9 @@ +variable "shared_hcl" {} +output "shared_hcl" { + value = var.shared_hcl +} + +variable "shared_tfvars" {} +output "shared_tfvars" { + value = var.shared_tfvars +} diff --git a/test/fixtures/units-reading/reading-hcl-and-tfvars/terragrunt.hcl b/test/fixtures/units-reading/reading-hcl-and-tfvars/terragrunt.hcl new file mode 100644 index 0000000000..fd8a3eced6 --- /dev/null +++ b/test/fixtures/units-reading/reading-hcl-and-tfvars/terragrunt.hcl @@ -0,0 +1,9 @@ +locals { + shared_hcl = read_terragrunt_config(find_in_parent_folders("shared.hcl")).locals + shared_tfvars = read_tfvars_file(find_in_parent_folders("shared.tfvars")) +} + +inputs = { + shared_hcl = local.shared_hcl + shared_tfvars = local.shared_tfvars +} diff --git a/test/fixtures/units-reading/reading-hcl/main.tf b/test/fixtures/units-reading/reading-hcl/main.tf new file mode 100644 index 0000000000..05d8c59b82 --- /dev/null +++ b/test/fixtures/units-reading/reading-hcl/main.tf @@ -0,0 +1,4 @@ +variable "shared" {} +output "shared" { + value = var.shared +} diff --git a/test/fixtures/units-reading/reading-hcl/terragrunt.hcl b/test/fixtures/units-reading/reading-hcl/terragrunt.hcl new file mode 100644 index 0000000000..06fab2f1c6 --- /dev/null +++ b/test/fixtures/units-reading/reading-hcl/terragrunt.hcl @@ -0,0 +1,7 @@ +locals { + shared = read_terragrunt_config(find_in_parent_folders("shared.hcl")).locals +} + +inputs = { + shared = local.shared +} diff --git a/test/fixtures/units-reading/reading-json/main.tf b/test/fixtures/units-reading/reading-json/main.tf new file mode 100644 index 0000000000..05d8c59b82 --- /dev/null +++ b/test/fixtures/units-reading/reading-json/main.tf @@ -0,0 +1,4 @@ +variable "shared" {} +output "shared" { + value = var.shared +} diff --git a/test/fixtures/units-reading/reading-json/terragrunt.hcl b/test/fixtures/units-reading/reading-json/terragrunt.hcl new file mode 100644 index 0000000000..11d8df1783 --- /dev/null +++ b/test/fixtures/units-reading/reading-json/terragrunt.hcl @@ -0,0 +1,7 @@ +locals { + shared = jsondecode(file(mark_as_read(find_in_parent_folders("shared.json")))) +} + +inputs = { + shared = local.shared +} diff --git a/test/fixtures/units-reading/reading-sops/main.tf b/test/fixtures/units-reading/reading-sops/main.tf new file mode 100644 index 0000000000..05d8c59b82 --- /dev/null +++ b/test/fixtures/units-reading/reading-sops/main.tf @@ -0,0 +1,4 @@ +variable "shared" {} +output "shared" { + value = var.shared +} diff --git a/test/fixtures/units-reading/reading-sops/terragrunt.hcl b/test/fixtures/units-reading/reading-sops/terragrunt.hcl new file mode 100644 index 0000000000..3e8310fa14 --- /dev/null +++ b/test/fixtures/units-reading/reading-sops/terragrunt.hcl @@ -0,0 +1,7 @@ +locals { + shared = sops_decrypt_file(find_in_parent_folders("secrets.txt")) +} + +inputs = { + shared = local.shared +} diff --git a/test/fixtures/units-reading/reading-tfvars/main.tf b/test/fixtures/units-reading/reading-tfvars/main.tf new file mode 100644 index 0000000000..05d8c59b82 --- /dev/null +++ b/test/fixtures/units-reading/reading-tfvars/main.tf @@ -0,0 +1,4 @@ +variable "shared" {} +output "shared" { + value = var.shared +} diff --git a/test/fixtures/units-reading/reading-tfvars/terragrunt.hcl b/test/fixtures/units-reading/reading-tfvars/terragrunt.hcl new file mode 100644 index 0000000000..4c6441f83f --- /dev/null +++ b/test/fixtures/units-reading/reading-tfvars/terragrunt.hcl @@ -0,0 +1,7 @@ +locals { + shared = read_tfvars_file(find_in_parent_folders("shared.tfvars")) +} + +inputs = { + shared = local.shared +} diff --git a/test/fixtures/units-reading/secrets.txt b/test/fixtures/units-reading/secrets.txt new file mode 100644 index 0000000000..37426edfcd --- /dev/null +++ b/test/fixtures/units-reading/secrets.txt @@ -0,0 +1,21 @@ +{ + "data": "ENC[AES256_GCM,data:w2jDRJR9BeIMSKE4+qnKWhfM,iv:08ACLYrUGtWriOV/ua4X6NZt57VmiTmAcnxB5V+8AUc=,tag:cVdkIO4EXAmyV3y7n/zbiA==,type:str]", + "sops": { + "kms": null, + "gcp_kms": null, + "azure_kv": null, + "hc_vault": null, + "age": null, + "lastmodified": "2021-12-17T18:38:13Z", + "mac": "ENC[AES256_GCM,data:8lPZmY8YgA0DqPRxLC9hVoRUXmbzaXgUBv3MHTm4iK44/6URIgJBUnPFPUbwIN7xbIgXd+QPQEMvfsmifqXorynGEwt2WtMKCPANg+2Ctf2KMmj7fGpe3HIlRhQiixip7/xzrIMbSdIRMS098D42JTvOIFNbWVQhByfN64AnDJY=,iv:wtouC/mWjhFwiJKDS6+5LqnQMcAeejElXLaL3H15jbY=,tag:6Bmemr2BMgShaMO3v4uiXw==,type:str]", + "pgp": [ + { + "created_at": "2021-12-17T18:38:12Z", + "enc": "-----BEGIN PGP MESSAGE-----\n\nhQEMA0sXzMgpEabgAQf+KHsPp4Pp8YNtG7ChRpZO2qB/bFncWtAF9evO+RjAEahb\nM+hzxkB5KDUSMYs0aeWeOrOqYPrjPPJxCspZtQhy8/qrC064kA7gq2PWhYAqGcKP\ntnPI8D0SYDZBgoyHRqFuuD5TZio8swE89SxphftL0W3KkHay7WKQHj/cFqNoISNl\nn0XeCgbacIwo5WxWz1qNFvaeo0rFFFhIhbfaegx/SWwUi1y6WK7sB0QobMRwXHj+\nORiUWVvx/fCIMCaerPN/SjIA/DgzbZ3DWaixYXpW85Ipz7myu/zUQcWnWcGXnMRQ\nERMYc6GyyLHwjZN1XuvXdPXvAt6vvaH4w5U9kW2l19JeAZXkcM14ivDoGwY1oLcX\n4d2/MAS7vM7SgmcPBGmpNsJJgkWTgoc8qeFtu9u3e4e9pR4+dcJCbGQLQ5RiyM2Z\nsyHjL6em/j4JLdtbM16orP6Q3oEPelphG7sxbDXBeA==\n=6u1S\n-----END PGP MESSAGE-----\n", + "fp": "3EF98802EEDCAF0C688B81F419546E0C123C664E" + } + ], + "unencrypted_suffix": "_unencrypted", + "version": "3.7.1" + } +} \ No newline at end of file diff --git a/test/fixtures/units-reading/shared.hcl b/test/fixtures/units-reading/shared.hcl new file mode 100644 index 0000000000..d5f16bab7d --- /dev/null +++ b/test/fixtures/units-reading/shared.hcl @@ -0,0 +1,3 @@ +locals { + shared_hcl = "value" +} diff --git a/test/fixtures/units-reading/shared.json b/test/fixtures/units-reading/shared.json new file mode 100644 index 0000000000..ead8846178 --- /dev/null +++ b/test/fixtures/units-reading/shared.json @@ -0,0 +1,3 @@ +{ + "key": "value" +} diff --git a/test/fixtures/units-reading/shared.tfvars b/test/fixtures/units-reading/shared.tfvars new file mode 100644 index 0000000000..0ffcc1d995 --- /dev/null +++ b/test/fixtures/units-reading/shared.tfvars @@ -0,0 +1 @@ +shared_tfvars = "value" diff --git a/test/fixtures/units-reading/test_pgp_key.asc b/test/fixtures/units-reading/test_pgp_key.asc new file mode 100644 index 0000000000..e42619f652 --- /dev/null +++ b/test/fixtures/units-reading/test_pgp_key.asc @@ -0,0 +1,87 @@ +-----BEGIN PGP PRIVATE KEY BLOCK----- + +lQOYBF69y60BCAC0Ton9jmk5/ZV3yKi2o0rY22G41dh9dLSoxTRJNKsKeSJePHK0 +qtrHriPjnHxFLXrXfYiv6FkdkPv9dhmKjMf1op/RRw0uTDZ6CsQd5C8nT9LBPiB6 +JsO89+GCwoVsPQGGVgqq7PWSbA8fJkAPBNvdQvxJnSbm4EJTcaU7XZmmY/eZnhPt +pho66X3PIQSx8Xm9Dc7jHo/7bBCoEV15OdFYi44imDW9sCdCpYTVzMF2ZRtXGylQ +U6P5Iu3SEMTnn4jwPpyKllT4qRFdqMQ4GvqE47uRb+k/FDXP8UC6h/5GNHfbQtEO +CMngw5vJciLEUWW1e97baBPiccLXNaBXiecXABEBAAEAB/44Mkpb1qkBRAHz5XyA +AADx+d5JR41D/L3Z5CzZrCqrBvopONnfaWjq12GkLT+mJ/ijdSLHALnVtzxy0P5A +6oPgESGSjnyTM3m/K9/YGSiBLiXXyM9CgxZ6rR+CK7J9+72f6u1EPLqFOly0Lq3E +gJUuLxSGtQ5M4xSJAWCoUhHzg6rUP978RQXc4AgfkfXaq/ILGyOOa7VSz4NVs3cZ ++WtPqlFhy5AsoMoLvXrwqLhVa9QsCiW0dYScRPD9q29wRHnKDN2nSxTuHbWu55r+ +krevzk5gPOTdj61vz6/xD/rqNFakZDjFfZIu+srqjnLMVEPYudrH0buIAF9RFYPp +8x+BBADTH3Q5+b+n770aFBDldIjD+biEnjnw6A396RNX2YIPqWoz0b7lQSYBM+DI +TTm7OFmacH2DSYHxx/e2GGOOglbc2czPlWGyegSfZblwcPPT47uVY6ZWIXAsszF6 +opL/ahQ336FEl8Ws8vEjUoTXWFzF50gfmdC2xPYYmoxymmZ1IQQA2qIvbPausqO7 +tHk9Bco1MLLzP5JIAvuLUNHOj9H+bUXh5btkm2gZRIGglScljF6HT180bJ4gt07n +H1QNpyWKuH0h0YSTDlBjLPQvElQAvxMuvT7RAWagXQo8ew+2FHbrH9Bhlys1wcG3 ++h1S2G8M6TyY0dv98TvqHV9RG+YCHTcEAMQTpJBn+yelK8LXyVrw9OmH0qmQ9TOF +uJihLBSd7nen+l4a3yDpISk6hrb/q4AjpzMJ8fK17AEuH9OxUkO8vgtxefvnoDTe +GYwuhszZRbWLYVDHZEkEjiGbCiqZw5530tHShA/IdBc5LMd9fwsag4Bru0cKhyCN +oe1FvVcABEHXR+O0EFRlcnJhZ3J1bnQgVGVzdHOJAUwEEwEKADYWIQQ++YgC7tyv +DGiLgfQZVG4MEjxmTgUCXr3LrQIbAwQLCQgHBBUKCQgFFgIDAQACHgECF4AACgkQ +GVRuDBI8Zk76SwgAkfEmyr9qOtNXpD4m8SnCrg8EJRX584hJyAu08/uB+QQV7Xzj +Azns4oVGGpZYEB7sUd22ejw8BGe6OI4puL43fnpDcFj4UA7FAIzQTKNztWFci2Ho +L0IAwcdzSlJb6qREMJacit8G4Aayn/bqZSNF4cu0XipRlOtK28FXP1PQThlNtSJL +yPpYRyU6s2CLSeRRqbi1MJ4KgKcnfTvv/4+VYQ6y7/rsDnXDA0nLgzYKMqtNdBPY +r4u7kZsGcWP8w9xQDWNJtOqj8NiRSAtnglqiYTHuEQsRjYGjH+vHqnGfw7dcK4+3 +Rlc7ZPGON2NCmJ6OajedIxZR6yNiDwj1AfuXxZ0DmARevcutAQgAtV1FnBgLKuqg +BVs09UbyrFuVJIx6M4e0IFac4bVzyfGJKjmxHx2n7D8slTZs+lcBNDy+CseAsAuc +ixv/jVAsl6saip5nUIAMmHkPkPM+BVAuqBRWoYWGFw3TL83MaYAgBkSkurx+Koxv +w2bTumxrb9xsyiy7Bn7kaIyZU/nN8Xb0xf6QC+y7ckzaI6ZT4utrtJGXya4Cu1rp +qUUYs/4ssWjZS7KuRwu8ijNwq/Zl0920IXH/sX/gUhL2AO5sgExC+52yjTTvhlIb +ee8tjMB4JT6Hq1RS9JGkNqcUOvJ99WlUoOtZUJU9xAN4eyPUy+r7v9wxcwoguanC +LWaKSNe9hQARAQABAAf8CToRuLNtHLBWFspWmI8/o67Ubh5qzc9UGycSO9YVlMsM +fZD00WPwBJq22RqCkyk0/vmDePHrD/RvRjvU6wAOsg1QQDLMh8cT8k01Elya+v3i +zxtFyFmOLQMU4O7j547PUkemEnf/eokC20U9H/60Z+WmGpJfAMwY27bMytMVJqPJ +gL1BNo9l0HGSTkU7mMqq48MsozTJvmCYbrVeKhnHdpAFHHSNQ9hGzgpZkpTisDk7 +qDIHhF1Nv7IRZgX4OGUbj1hBk63ao1TclhbB8d7gitvTODbMxFOF9crm7ttWtq/C +CIBM9X5ilufEnuN2eV4LxLHeghQOGsJhl/FB8gov+wQAx/Ja/2WFzHoEc5evJtNU +ifKpaIAp4Qj673dx21Vzr43qnuNNxLuG53FvgRpfXhVRyzWmNnJByQ3NRVUv4J1j +GXymlPPPzmAbML8874zShMnMcd1+UCZ+0dgFeB6CetVySExO0p+qUW+fioP5jrJk +spNxtY01RE5AUSWUeQN3sdcEAOg1T1cUIEq/dgPLDQE3mPQta7fJeiDddheNgQRp +xKHxDKSpWd1RtmiUxG0cT3z68M8aRcL/X+q731WGadNhM+yp4xg6wVTpL8USdLZr +qqiuvMYqowryZOdvPUP8OE1lQwtWizCFOoNL+yJyKVzt7+Z0CrkE8s3li68sLMeg +z5gDBACflcuTWLMNt3buo/31YrNWLDRxDMdKpNZ0Tpj+Kxda9+GjRGWdMZHwOsIO +WhGnftxtbKSWu2+PabFBchiwLC1r4WHMFy/fxFFhufJtYI/c58kRd+9I0vw6/JQx +WvGunELyeTnNu3u+uvagUSchBhNln5hZZOtpaBaDhNngm5DXM0g+iQE2BBgBCgAg +FiEEPvmIAu7crwxoi4H0GVRuDBI8Zk4FAl69y60CGwwACgkQGVRuDBI8Zk4gogf/ +SyXhch/Ep9ESW+Zx7T+ImfQAVPBmka0NNCEvSF5i/ghTm700Iv/sLcRqhSJDiKLb +1imeqVN6khmr6+d21uCqqgiv/6X4w6zmxH9h4uPMv3H6WnHi40kDUawR7hFctNLX +twArH+xCX5MJgzhqAP5Yzgsk5XLwzgjCuXz06RqtmZVW4ofOP4GWBT24Gg4TyUF9 +Ibl1/QLKeYOarD4a/PcEfHhJyvnAmD68uIFr1gABJJneW8vvO4OPjLcptZMIZ1Nf +xqUuLBAJ71OI3RSSowpI3qCpFhH2j5vICM7jX2gIL9PQEGGg2ljNJoorJIEjYZzS +0S9IMjn4Mj7CYtAZarnIQw== +=fO3n +-----END PGP PRIVATE KEY BLOCK----- +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQENBF69y60BCAC0Ton9jmk5/ZV3yKi2o0rY22G41dh9dLSoxTRJNKsKeSJePHK0 +qtrHriPjnHxFLXrXfYiv6FkdkPv9dhmKjMf1op/RRw0uTDZ6CsQd5C8nT9LBPiB6 +JsO89+GCwoVsPQGGVgqq7PWSbA8fJkAPBNvdQvxJnSbm4EJTcaU7XZmmY/eZnhPt +pho66X3PIQSx8Xm9Dc7jHo/7bBCoEV15OdFYi44imDW9sCdCpYTVzMF2ZRtXGylQ +U6P5Iu3SEMTnn4jwPpyKllT4qRFdqMQ4GvqE47uRb+k/FDXP8UC6h/5GNHfbQtEO +CMngw5vJciLEUWW1e97baBPiccLXNaBXiecXABEBAAG0EFRlcnJhZ3J1bnQgVGVz +dHOJAUwEEwEKADYWIQQ++YgC7tyvDGiLgfQZVG4MEjxmTgUCXr3LrQIbAwQLCQgH +BBUKCQgFFgIDAQACHgECF4AACgkQGVRuDBI8Zk76SwgAkfEmyr9qOtNXpD4m8SnC +rg8EJRX584hJyAu08/uB+QQV7XzjAzns4oVGGpZYEB7sUd22ejw8BGe6OI4puL43 +fnpDcFj4UA7FAIzQTKNztWFci2HoL0IAwcdzSlJb6qREMJacit8G4Aayn/bqZSNF +4cu0XipRlOtK28FXP1PQThlNtSJLyPpYRyU6s2CLSeRRqbi1MJ4KgKcnfTvv/4+V +YQ6y7/rsDnXDA0nLgzYKMqtNdBPYr4u7kZsGcWP8w9xQDWNJtOqj8NiRSAtnglqi +YTHuEQsRjYGjH+vHqnGfw7dcK4+3Rlc7ZPGON2NCmJ6OajedIxZR6yNiDwj1AfuX +xbkBDQRevcutAQgAtV1FnBgLKuqgBVs09UbyrFuVJIx6M4e0IFac4bVzyfGJKjmx +Hx2n7D8slTZs+lcBNDy+CseAsAucixv/jVAsl6saip5nUIAMmHkPkPM+BVAuqBRW +oYWGFw3TL83MaYAgBkSkurx+Koxvw2bTumxrb9xsyiy7Bn7kaIyZU/nN8Xb0xf6Q +C+y7ckzaI6ZT4utrtJGXya4Cu1rpqUUYs/4ssWjZS7KuRwu8ijNwq/Zl0920IXH/ +sX/gUhL2AO5sgExC+52yjTTvhlIbee8tjMB4JT6Hq1RS9JGkNqcUOvJ99WlUoOtZ +UJU9xAN4eyPUy+r7v9wxcwoguanCLWaKSNe9hQARAQABiQE2BBgBCgAgFiEEPvmI +Au7crwxoi4H0GVRuDBI8Zk4FAl69y60CGwwACgkQGVRuDBI8Zk4gogf/SyXhch/E +p9ESW+Zx7T+ImfQAVPBmka0NNCEvSF5i/ghTm700Iv/sLcRqhSJDiKLb1imeqVN6 +khmr6+d21uCqqgiv/6X4w6zmxH9h4uPMv3H6WnHi40kDUawR7hFctNLXtwArH+xC +X5MJgzhqAP5Yzgsk5XLwzgjCuXz06RqtmZVW4ofOP4GWBT24Gg4TyUF9Ibl1/QLK +eYOarD4a/PcEfHhJyvnAmD68uIFr1gABJJneW8vvO4OPjLcptZMIZ1NfxqUuLBAJ +71OI3RSSowpI3qCpFhH2j5vICM7jX2gIL9PQEGGg2ljNJoorJIEjYZzS0S9IMjn4 +Mj7CYtAZarnIQw== +=dO7W +-----END PGP PUBLIC KEY BLOCK----- diff --git a/test/integartion_units_reading_test.go b/test/integartion_units_reading_test.go new file mode 100644 index 0000000000..245100af40 --- /dev/null +++ b/test/integartion_units_reading_test.go @@ -0,0 +1,162 @@ +package test_test + +import ( + "regexp" + "strings" + "testing" + + "github.com/gruntwork-io/terragrunt/test/helpers" + "github.com/gruntwork-io/terragrunt/util" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + testFixtureUnitsReading = "fixtures/units-reading/" +) + +func TestUnitsReading(t *testing.T) { + t.Parallel() + + cleanupTerraformFolder(t, testFixtureUnitsReading) + + tc := []struct { + name string + unitsReading []string + unitsExcluding []string + unitsIncluding []string + expectedUnits []string + }{ + { + name: "empty", + unitsReading: []string{}, + expectedUnits: []string{ + "reading-from-tf", + "reading-hcl", + "reading-hcl-and-tfvars", + "reading-json", + "reading-sops", + "reading-tfvars", + }, + }, + { + name: "reading_hcl", + unitsReading: []string{ + "shared.hcl", + }, + expectedUnits: []string{ + "reading-hcl", + "reading-hcl-and-tfvars", + }, + }, + { + name: "reading_tfvars", + unitsReading: []string{ + "shared.tfvars", + }, + expectedUnits: []string{ + "reading-tfvars", + "reading-hcl-and-tfvars", + }, + }, + { + name: "reading_json", + unitsReading: []string{ + "shared.json", + }, + expectedUnits: []string{ + "reading-from-tf", + "reading-json", + }, + }, + { + name: "reading_sops", + unitsReading: []string{ + "secrets.txt", + }, + expectedUnits: []string{ + "reading-sops", + }, + }, + { + name: "reading_from_hcl_with_exclude", + unitsReading: []string{ + "shared.hcl", + }, + unitsExcluding: []string{ + "reading-hcl-and-tfvars", + }, + expectedUnits: []string{ + "reading-hcl", + }, + }, + { + name: "reading_from_hcl_with_include", + unitsReading: []string{ + "shared.hcl", + }, + unitsIncluding: []string{ + "reading-tfvars", + }, + expectedUnits: []string{ + "reading-hcl", + "reading-hcl-and-tfvars", + "reading-tfvars", + }, + }, + { + name: "reading_from_hcl_with_include_and_exclude", + unitsReading: []string{ + "shared.hcl", + "shared.tfvars", + }, + unitsIncluding: []string{ + "reading-tfvars", + }, + unitsExcluding: []string{ + "reading-hcl-and-tfvars", + }, + expectedUnits: []string{ + "reading-hcl", + "reading-tfvars", + }, + }, + } + + includedLogEntryRegex := regexp.MustCompile(`=> Module ./([^ ]+) \(excluded: false`) + + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + tmpEnvPath := helpers.CopyEnvironment(t, testFixtureUnitsReading) + rootPath := util.JoinPath(tmpEnvPath, testFixtureUnitsReading) + + cmd := "terragrunt run-all plan --terragrunt-non-interactive --terragrunt-log-level debug --terragrunt-working-dir " + rootPath + + for _, unit := range tt.unitsReading { + cmd = cmd + " --terragrunt-queue-include-units-reading " + unit + } + + for _, unit := range tt.unitsIncluding { + cmd = cmd + " --terragrunt-include-dir " + unit + } + + for _, unit := range tt.unitsExcluding { + cmd = cmd + " --terragrunt-exclude-dir " + unit + } + + _, stderr, err := helpers.RunTerragruntCommandWithOutput(t, cmd) + require.NoError(t, err) + + includedUnits := []string{} + for _, line := range strings.Split(stderr, "\n") { + if includedLogEntryRegex.MatchString(line) { + includedUnits = append(includedUnits, includedLogEntryRegex.FindStringSubmatch(line)[1]) + } + } + + assert.ElementsMatch(t, tt.expectedUnits, includedUnits) + }) + } +} diff --git a/test/integration_aws_test.go b/test/integration_aws_test.go index 065aad92c2..d50aab1bbb 100644 --- a/test/integration_aws_test.go +++ b/test/integration_aws_test.go @@ -759,74 +759,79 @@ func TestAwsAssumeRoleDuration(t *testing.T) { assert.Contains(t, output, "no changes are needed.") } -func TestAwsAssumeRoleWebIdentityEnv(t *testing.T) { - t.Parallel() - - assumeRole := os.Getenv("AWS_TEST_S3_ASSUME_ROLE") - tokenEnvVar := os.Getenv("AWS_TEST_S3_IDENTITY_TOKEN_VAR") - if tokenEnvVar == "" { - t.Skip("Missing required env var AWS_TEST_S3_IDENTITY_TOKEN_VAR") - return +func TestAwsAssumeRoleWebIdentityFile(t *testing.T) { + if os.Getenv("CIRCLECI") != "true" { + t.Skip("Skipping test because it requires valid CircleCI OIDC credentials to work") } - tmpEnvPath := helpers.CopyEnvironment(t, testFixtureAssumeRoleWebIdentityEnv) - helpers.CleanupTerraformFolder(t, tmpEnvPath) - testPath := util.JoinPath(tmpEnvPath, testFixtureAssumeRoleWebIdentityEnv) + // These tests need to be run without the static key + secret + // used by most AWS tests here. + t.Setenv("AWS_ACCESS_KEY_ID", "") + os.Unsetenv("AWS_ACCESS_KEY_ID") + t.Setenv("AWS_SECRET_ACCESS_KEY", "") + os.Unsetenv("AWS_SECRET_ACCESS_KEY") - originalTerragruntConfigPath := util.JoinPath(testFixtureAssumeRoleWebIdentityEnv, "terragrunt.hcl") + tmpEnvPath := helpers.CopyEnvironment(t, testFixtureAssumeRoleWebIdentityFile) + cleanupTerraformFolder(t, tmpEnvPath) + testPath := util.JoinPath(tmpEnvPath, testFixtureAssumeRoleWebIdentityFile) + + originalTerragruntConfigPath := util.JoinPath(testFixtureAssumeRoleWebIdentityFile, "terragrunt.hcl") tmpTerragruntConfigFile := util.JoinPath(testPath, "terragrunt.hcl") s3BucketName := "terragrunt-test-bucket-" + strings.ToLower(helpers.UniqueID()) - defer helpers.DeleteS3Bucket(t, helpers.TerraformRemoteStateS3Region, s3BucketName, options.WithIAMRoleARN(assumeRole), options.WithIAMWebIdentityToken(os.Getenv(tokenEnvVar))) + role := os.Getenv("AWS_TEST_OIDC_ROLE_ARN") + require.NotEmpty(t, role) + token := os.Getenv("CIRCLE_OIDC_TOKEN_V2") + require.NotEmpty(t, token) + + tokenFile := t.TempDir() + "/oidc-token" + require.NoError(t, os.WriteFile(tokenFile, []byte(token), 0400)) + + defer helpers.DeleteS3Bucket(t, helpers.TerraformRemoteStateS3Region, s3BucketName, options.WithIAMRoleARN(role), options.WithIAMWebIdentityToken(token)) helpers.CopyAndFillMapPlaceholders(t, originalTerragruntConfigPath, tmpTerragruntConfigFile, map[string]string{ - "__FILL_IN_BUCKET_NAME__": s3BucketName, - "__FILL_IN_REGION__": helpers.TerraformRemoteStateS3Region, - "__FILL_IN_ASSUME_ROLE__": assumeRole, - "__FILL_IN_IDENTITY_TOKEN_ENV_VAR__": tokenEnvVar, + "__FILL_IN_BUCKET_NAME__": s3BucketName, + "__FILL_IN_REGION__": helpers.TerraformRemoteStateS3Region, + "__FILL_IN_ASSUME_ROLE__": role, + "__FILL_IN_IDENTITY_TOKEN_FILE_PATH__": tokenFile, }) stdout := bytes.Buffer{} stderr := bytes.Buffer{} - err := helpers.RunTerragruntCommand(t, "terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir "+testPath, &stdout, &stderr) + err := helpers.RunTerragruntCommand(t, "terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-log-level debug --terragrunt-working-dir "+testPath, &stdout, &stderr) require.NoError(t, err) output := fmt.Sprintf("%s %s", stderr.String(), stdout.String()) assert.Contains(t, output, "Apply complete! Resources: 1 added, 0 changed, 0 destroyed.") } -func TestAwsAssumeRoleWebIdentityFile(t *testing.T) { - t.Parallel() - - tmpEnvPath := helpers.CopyEnvironment(t, testFixtureAssumeRoleWebIdentityFile) - helpers.CleanupTerraformFolder(t, tmpEnvPath) - testPath := util.JoinPath(tmpEnvPath, testFixtureAssumeRoleWebIdentityFile) - - originalTerragruntConfigPath := util.JoinPath(testFixtureAssumeRoleWebIdentityFile, "terragrunt.hcl") - tmpTerragruntConfigFile := util.JoinPath(testPath, "terragrunt.hcl") - s3BucketName := "terragrunt-test-bucket-" + strings.ToLower(helpers.UniqueID()) +func TestAwsAssumeRoleWebIdentityFlag(t *testing.T) { + if os.Getenv("CIRCLECI") != "true" { + t.Skip("Skipping test because it requires valid CircleCI OIDC credentials to work") + } - assumeRole := os.Getenv("AWS_TEST_S3_ASSUME_ROLE") - tokenFilePath := os.Getenv("AWS_TEST_S3_IDENTITY_TOKEN_FILE_PATH") + // These tests need to be run without the static key + secret + // used by most AWS tests here. + t.Setenv("AWS_ACCESS_KEY_ID", "") + os.Unsetenv("AWS_ACCESS_KEY_ID") + t.Setenv("AWS_SECRET_ACCESS_KEY", "") + os.Unsetenv("AWS_SECRET_ACCESS_KEY") - defer helpers.DeleteS3Bucket(t, helpers.TerraformRemoteStateS3Region, s3BucketName, options.WithIAMRoleARN(assumeRole), options.WithIAMWebIdentityToken(tokenFilePath)) + tmp := t.TempDir() - helpers.CopyAndFillMapPlaceholders(t, originalTerragruntConfigPath, tmpTerragruntConfigFile, map[string]string{ - "__FILL_IN_BUCKET_NAME__": s3BucketName, - "__FILL_IN_REGION__": helpers.TerraformRemoteStateS3Region, - "__FILL_IN_ASSUME_ROLE__": assumeRole, - "__FILL_IN_IDENTITY_TOKEN_FILE_PATH__": tokenFilePath, - }) + emptyTerragruntConfigPath := filepath.Join(tmp, "terragrunt.hcl") + require.NoError(t, os.WriteFile(emptyTerragruntConfigPath, []byte(""), 0400)) - stdout := bytes.Buffer{} - stderr := bytes.Buffer{} + emptyMainTFPath := filepath.Join(tmp, "main.tf") + require.NoError(t, os.WriteFile(emptyMainTFPath, []byte(""), 0400)) - err := helpers.RunTerragruntCommand(t, "terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir "+testPath, &stdout, &stderr) - require.NoError(t, err) + roleARN := os.Getenv("AWS_TEST_OIDC_ROLE_ARN") + require.NotEmpty(t, roleARN) + token := os.Getenv("CIRCLE_OIDC_TOKEN_V2") + require.NotEmpty(t, token) - output := fmt.Sprintf("%s %s", stderr.String(), stdout.String()) - assert.Contains(t, output, "Apply complete! Resources: 1 added, 0 changed, 0 destroyed.") + helpers.RunTerragrunt(t, "terragrunt apply --terragrunt-non-interactive --terragrunt-log-level debug --terragrunt-working-dir "+tmp+" --terragrunt-iam-role "+roleARN+" --terragrunt-iam-web-identity-token "+token) } // Regression testing for https://github.com/gruntwork-io/terragrunt/issues/906 @@ -1096,6 +1101,21 @@ func TestAwsReadTerragruntAuthProviderCmdWithSops(t *testing.T) { assert.Equal(t, "Welcome to SOPS! Edit this file as you please!", outputs["hello"].Value) } +func TestAwsReadTerragruntAuthProviderCmdWithOIDC(t *testing.T) { + t.Parallel() + + if os.Getenv("CIRCLECI") != "true" { + t.Skip("Skipping test because it requires valid CircleCI OIDC credentials to work") + } + + cleanupTerraformFolder(t, testFixtureAuthProviderCmd) + tmpEnvPath := helpers.CopyEnvironment(t, testFixtureAuthProviderCmd) + oidcPath := util.JoinPath(tmpEnvPath, testFixtureAuthProviderCmd, "oidc") + mockAuthCmd := filepath.Join(oidcPath, "mock-auth-cmd.sh") + + helpers.RunTerragrunt(t, fmt.Sprintf(`terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir %s --terragrunt-auth-provider-cmd %s`, oidcPath, mockAuthCmd)) +} + func TestAwsReadTerragruntConfigIamRole(t *testing.T) { t.Parallel()