From 940ec1de7f8421664f3c435ef6f2f3c8ecfebe29 Mon Sep 17 00:00:00 2001 From: Vincent De Smet Date: Tue, 28 Jan 2025 23:32:17 +0700 Subject: [PATCH] feat: Add --env/--component apply filter flags Quick speed win --- apply/apply.go | 83 +++++++++++++---------- apply/apply_test.go | 134 +++++++++++++++++++++++++++++++++++++- apply/golden_file_test.go | 4 +- cmd/apply.go | 22 ++++++- 4 files changed, 203 insertions(+), 40 deletions(-) diff --git a/apply/apply.go b/apply/apply.go index e8e40cae3..8f380db82 100644 --- a/apply/apply.go +++ b/apply/apply.go @@ -31,7 +31,7 @@ import ( ) // Apply will run a plan and apply all the changes to the current repo. -func Apply(fs afero.Fs, conf *v2.Config, tmpl *templates.T, upgrade bool) error { +func Apply(fs afero.Fs, conf *v2.Config, tmpl *templates.T, upgrade bool, envFilter *string, compFilter *string) error { if !upgrade { toolVersion, err := util.VersionString() if err != nil { @@ -46,69 +46,72 @@ func Apply(fs afero.Fs, conf *v2.Config, tmpl *templates.T, upgrade bool) error if err != nil { return errs.WrapUser(err, "unable to evaluate plan") } - err = applyRepo(fs, plan, tmpl.Repo, tmpl.Common) - if err != nil { - return errs.WrapUser(err, "unable to apply repo") - } - if plan.TravisCI.Enabled { - err = applyTree(fs, tmpl.TravisCI, tmpl.Common, "", plan.TravisCI) + if envFilter == nil { + err = applyRepo(fs, plan, tmpl.Repo, tmpl.Common) if err != nil { - return errs.WrapUser(err, "unable to apply travis ci") + return errs.WrapUser(err, "unable to apply repo") } - } - if plan.CircleCI.Enabled { - err = applyTree(fs, tmpl.CircleCI, tmpl.Common, "", plan.CircleCI) - if err != nil { - return errs.WrapUser(err, "unable to apply CircleCI") + if plan.TravisCI.Enabled { + err = applyTree(fs, tmpl.TravisCI, tmpl.Common, "", plan.TravisCI) + if err != nil { + return errs.WrapUser(err, "unable to apply travis ci") + } } - } - if plan.GitHubActionsCI.Enabled { - err = applyTree(fs, tmpl.GitHubActionsCI, tmpl.Common, ".github", plan.GitHubActionsCI) - if err != nil { - return errs.WrapUser(err, "unable to apply GitHub Actions CI") + if plan.CircleCI.Enabled { + err = applyTree(fs, tmpl.CircleCI, tmpl.Common, "", plan.CircleCI) + if err != nil { + return errs.WrapUser(err, "unable to apply CircleCI") + } } - } - if plan.Turbo.Enabled { - err = applyTree(fs, tmpl.TurboRoot, tmpl.Common, "", plan.Turbo) - if err != nil { - return errs.WrapUser(err, "unable to apply Turbo config") + if plan.GitHubActionsCI.Enabled { + err = applyTree(fs, tmpl.GitHubActionsCI, tmpl.Common, ".github", plan.GitHubActionsCI) + if err != nil { + return errs.WrapUser(err, "unable to apply GitHub Actions CI") + } } - } - tfBox := tmpl.Components[v2.ComponentKindTerraform] - err = applyAccounts(fs, plan, tfBox, tmpl.Common) - if err != nil { - return errs.WrapUser(err, "unable to apply accounts") - } + if plan.Turbo.Enabled { + err = applyTree(fs, tmpl.TurboRoot, tmpl.Common, "", plan.Turbo) + if err != nil { + return errs.WrapUser(err, "unable to apply Turbo config") + } + } - err = applyModules(fs, plan.Modules, tmpl.Module, tmpl.Common) - if err != nil { - return errs.WrapUser(err, "unable to apply modules") + err = applyModules(fs, plan.Modules, tmpl.Module, tmpl.Common) + if err != nil { + return errs.WrapUser(err, "unable to apply modules") + } + + tfBox := tmpl.Components[v2.ComponentKindTerraform] + err = applyAccounts(fs, plan, tfBox, tmpl.Common) + if err != nil { + return errs.WrapUser(err, "unable to apply accounts") + } } - pathModuleConfigs, err := applyEnvs(fs, plan, tmpl.Env, tmpl.Components, tmpl.Common) + pathModuleConfigs, err := applyEnvs(fs, plan, envFilter, compFilter, tmpl.Env, tmpl.Components, tmpl.Common) if err != nil { return errs.WrapUser(err, "unable to apply envs") } - if plan.Atlantis.Enabled { + if envFilter == nil && plan.Atlantis.Enabled { err = applyAtlantisConfig(fs, tmpl.Atlantis, tmpl.Common, "", &plan.Atlantis, pathModuleConfigs) if err != nil { return errs.WrapUser(err, "unable to apply Atlantis") } } - tfBox = tmpl.Components[v2.ComponentKindTerraform] + tfBox := tmpl.Components[v2.ComponentKindTerraform] err = applyGlobal(fs, plan.Global, tfBox, tmpl.Common) if err != nil { return errs.WrapUser(err, "unable to apply global") } - if plan.GitHubActionsCI.Enabled && plan.GitHubActionsCI.PreCommit.Enabled { + if envFilter == nil && plan.GitHubActionsCI.Enabled && plan.GitHubActionsCI.PreCommit.Enabled { // set up pre-commit config preCommit := plan.GitHubActionsCI.PreCommit err = applyTree(fs, tmpl.PreCommitRoot, tmpl.Common, "", preCommit) @@ -377,12 +380,17 @@ type PathModuleConfigs map[string]ModuleConfigMap func applyEnvs( fs afero.Fs, p *plan.Plan, + envFilter *string, + compFilter *string, envBox fs.FS, componentBoxes map[v2.ComponentKind]fs.FS, commonBox fs.FS) (pathModuleConfigs PathModuleConfigs, err error) { logrus.Debug("applying envs") pathModuleConfigs = make(PathModuleConfigs) for env, envPlan := range p.Envs { + if envFilter != nil && *envFilter != env { + continue + } logrus.Debugf("applying %s", env) path := fmt.Sprintf("%s/envs/%s", util.RootPath, env) err = fs.MkdirAll(path, 0755) @@ -395,6 +403,9 @@ func applyEnvs( } reg := registry.NewClient(nil, nil) for component, componentPlan := range envPlan.Components { + if compFilter != nil && *compFilter != component { + continue + } path = fmt.Sprintf("%s/envs/%s/%s", util.RootPath, env, component) err = fs.MkdirAll(path, 0755) if err != nil { diff --git a/apply/apply_test.go b/apply/apply_test.go index d87f6d595..eb6e10e4b 100644 --- a/apply/apply_test.go +++ b/apply/apply_test.go @@ -392,7 +392,7 @@ version: 2 r.NoError(e) r.Len(w, 0) - e = Apply(fs, c, templates.Templates, false) + e = Apply(fs, c, templates.Templates, false, nil, nil) r.NoError(e) } @@ -830,3 +830,135 @@ func Test_dropDirectorySuffix(t *testing.T) { }) } } + +func TestApplyEnvsWithFilter(t *testing.T) { + r := require.New(t) + tmpl := templates.Templates + dest, d, err := util.TestFs() + r.NoError(err) + defer os.RemoveAll(d) + + conf := &v2.Config{ + Defaults: v2.Defaults{ + Common: v2.Common{ + TerraformVersion: util.Ptr("0.12.0"), + Owner: util.Ptr("owner"), + Project: util.Ptr("project"), + Backend: &v2.Backend{ + Kind: util.Ptr(string(plan.BackendKindS3)), + Bucket: util.Ptr("bucket"), + Region: util.Ptr("region"), + Profile: util.Ptr("profile"), + }, + }, + }, + Envs: map[string]v2.Env{ + "env1": { + Components: map[string]v2.Component{ + "component1": {}, + }, + }, + "env2": { + Components: map[string]v2.Component{ + "component2": {}, + }, + }, + }, + } + + plan, err := plan.Eval(conf) + r.NoError(err) + envFilter := "env1" + + pathModuleConfigs, err := applyEnvs(dest, plan, &envFilter, nil, tmpl.Env, tmpl.Components, tmpl.Common) + r.NoError(err) + r.NotNil(pathModuleConfigs) + _, err = dest.Stat("terraform/envs/env1") + r.NoError(err) + _, err = dest.Stat("terraform/envs/env2") + r.Error(err) +} + +func TestApplyEnvsWithCompFilter(t *testing.T) { + r := require.New(t) + tmpl := templates.Templates + + conf := &v2.Config{ + Defaults: v2.Defaults{ + Common: v2.Common{ + TerraformVersion: util.Ptr("0.12.0"), + Owner: util.Ptr("owner"), + Project: util.Ptr("project"), + Backend: &v2.Backend{ + Kind: util.Ptr(string(plan.BackendKindS3)), + Bucket: util.Ptr("bucket"), + Region: util.Ptr("region"), + Profile: util.Ptr("profile"), + }, + }, + }, + Envs: map[string]v2.Env{ + "env1": { + Components: map[string]v2.Component{ + "component1": {}, + "component2": {}, + }, + }, + "env2": { + Components: map[string]v2.Component{ + "component2": {}, + }, + }, + }, + } + + plan, err := plan.Eval(conf) + r.NoError(err) + + tests := []struct { + envFilter *string + compFilter *string + expectedDirs []string + unexpectedDirs []string + }{ + {util.Ptr("env1"), util.Ptr("component1"), []string{"terraform/envs/env1/component1"}, []string{"terraform/envs/env1/component2", "terraform/envs/env2"}}, + {util.Ptr("env1"), util.Ptr("component2"), []string{"terraform/envs/env1/component2"}, []string{"terraform/envs/env1/component1", "terraform/envs/env2"}}, + {util.Ptr("env2"), util.Ptr("component2"), []string{"terraform/envs/env2/component2"}, []string{"terraform/envs/env1"}}, + {util.Ptr("env1"), nil, []string{"terraform/envs/env1/component1", "terraform/envs/env1/component2"}, []string{"terraform/envs/env2"}}, + {nil, nil, []string{"terraform/envs/env1/component1", "terraform/envs/env1/component2", "terraform/envs/env2/component2"}, []string{}}, + } + + for _, tt := range tests { + var testName string + if tt.envFilter == nil { + testName = "all" + } else { + testName = *tt.envFilter + if tt.compFilter != nil { + testName += "_" + *tt.compFilter + } else { + testName += "_all" + } + } + + t.Run(testName, func(t *testing.T) { + dest, d, err := util.TestFs() + r.NoError(err) + defer os.RemoveAll(d) + + pathModuleConfigs, err := applyEnvs(dest, plan, tt.envFilter, tt.compFilter, tmpl.Env, tmpl.Components, tmpl.Common) + r.NoError(err) + r.NotNil(pathModuleConfigs) + + for _, dir := range tt.expectedDirs { + _, err = dest.Stat(dir) + r.NoError(err) + } + + for _, dir := range tt.unexpectedDirs { + _, err = dest.Stat(dir) + r.Error(err) + } + }) + } +} diff --git a/apply/golden_file_test.go b/apply/golden_file_test.go index ea0aa416b..fe6736155 100644 --- a/apply/golden_file_test.go +++ b/apply/golden_file_test.go @@ -108,7 +108,7 @@ func TestIntegration(t *testing.T) { r.NoError(e) r.Len(w, 0) - e = apply.Apply(testdataFs, conf, templates.Templates, true) + e = apply.Apply(testdataFs, conf, templates.Templates, true, nil, nil) r.NoError(e) } else { fs, _, e := util.TestFs() @@ -152,7 +152,7 @@ func TestIntegration(t *testing.T) { r.NoError(e) r.Len(w, 0) - e = apply.Apply(fs, conf, templates.Templates, true) + e = apply.Apply(fs, conf, templates.Templates, true, nil, nil) r.NoError(e) r.NoError(afero.Walk(testdataFs, ".", func(path string, info os.FileInfo, err error) error { diff --git a/cmd/apply.go b/cmd/apply.go index 5693acd83..57d766d67 100644 --- a/cmd/apply.go +++ b/cmd/apply.go @@ -1,14 +1,21 @@ package cmd import ( + "fmt" + "github.com/chanzuckerberg/fogg/apply" "github.com/chanzuckerberg/fogg/templates" "github.com/spf13/cobra" ) +var environment string +var component string + func init() { applyCmd.Flags().StringP("config", "c", "fogg.yml", "Use this to override the fogg config file.") applyCmd.Flags().BoolP("upgrade", "u", false, "Use this when running a new version of fogg") + applyCmd.Flags().StringVarP(&environment, "env", "e", "", "Limit apply to specific environment") + applyCmd.Flags().StringVarP(&component, "component", "f", "", "Limit apply to specific component (requires env flag)") rootCmd.AddCommand(applyCmd) } @@ -36,6 +43,19 @@ var applyCmd = &cobra.Command{ return e } + var envFilter *string + if environment != "" { + envFilter = &environment + } + + var compFilter *string + if component != "" { + compFilter = &component + if envFilter == nil { + return fmt.Errorf("component flag requires env flag") + } + } + // check that we are at root of initialized git repo openGitOrExit(fs) @@ -48,7 +68,7 @@ var applyCmd = &cobra.Command{ } // apply - e = apply.Apply(fs, config, templates.Templates, upgrade) + e = apply.Apply(fs, config, templates.Templates, upgrade, envFilter, compFilter) return e },