From e1c00324f97c6c3e65a49faebb86a91469aa59a3 Mon Sep 17 00:00:00 2001 From: Kris Budde Date: Sun, 29 Dec 2024 19:56:06 +0100 Subject: [PATCH 1/3] make app-data files globable --- internal/myks/application.go | 7 ++++--- internal/myks/environment.go | 4 ++-- internal/myks/globe.go | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/internal/myks/application.go b/internal/myks/application.go index b69278f6..22bcefbc 100644 --- a/internal/myks/application.go +++ b/internal/myks/application.go @@ -119,13 +119,14 @@ func (a *Application) collectDataFiles() { a.yttDataFiles = append(a.yttDataFiles, environmentDataFiles...) protoDataFile := filepath.Join(a.Prototype, a.e.g.ApplicationDataFileName) - if ok, err := isExist(protoDataFile); ok && err == nil { - a.yttDataFiles = append(a.yttDataFiles, protoDataFile) + if files, err := filepath.Glob(protoDataFile); err == nil { + a.yttDataFiles = append(a.yttDataFiles, files...) } protoOverrideDataFiles := a.e.collectBySubpath(filepath.Join(a.e.g.PrototypeOverrideDir, a.prototypeDirName(), a.e.g.ApplicationDataFileName)) a.yttDataFiles = append(a.yttDataFiles, protoOverrideDataFiles...) - overrideDataFiles := append(protoOverrideDataFiles, a.e.collectBySubpath(filepath.Join(a.e.g.AppsDir, a.Name, a.e.g.ApplicationDataFileName))...) + + overrideDataFiles := a.e.collectBySubpath(filepath.Join(a.e.g.AppsDir, a.Name, a.e.g.ApplicationDataFileName)) a.yttDataFiles = append(a.yttDataFiles, overrideDataFiles...) } diff --git a/internal/myks/environment.go b/internal/myks/environment.go index 3fa61a84..a4d849fc 100644 --- a/internal/myks/environment.go +++ b/internal/myks/environment.go @@ -359,8 +359,8 @@ func (e *Environment) collectBySubpath(subpath string) []string { } currentPath = filepath.Join(currentPath, level) item := filepath.Join(currentPath, subpath) - if ok, err := isExist(item); ok && err == nil { - items = append(items, item) + if files, err := filepath.Glob(item); err == nil { + items = append(items, files...) } } return items diff --git a/internal/myks/globe.go b/internal/myks/globe.go index 27f72407..5a7cbc27 100644 --- a/internal/myks/globe.go +++ b/internal/myks/globe.go @@ -46,7 +46,7 @@ type Globe struct { // Data values schema file name DataSchemaFileName string `default:"data-schema.ytt.yaml"` // Application data file name - ApplicationDataFileName string `default:"app-data.ytt.yaml"` + ApplicationDataFileName string `default:"app-data.*.yaml"` // Environment data file name EnvironmentDataFileName string `default:"env-data.ytt.yaml"` // Rendered environment data file name From f6fbd055119333ef749ba39ffef042d26da3edb2 Mon Sep 17 00:00:00 2001 From: Kris Budde Date: Sun, 29 Dec 2024 20:08:08 +0100 Subject: [PATCH 2/3] adjusted tests --- internal/myks/smart_mode.go | 13 ++++++++++--- internal/myks/smart_mode_test.go | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/internal/myks/smart_mode.go b/internal/myks/smart_mode.go index fde2b65b..288260b6 100644 --- a/internal/myks/smart_mode.go +++ b/internal/myks/smart_mode.go @@ -66,6 +66,13 @@ func (g *Globe) runSmartMode(changedFiles ChangedFiles) EnvAppMap { return regexp.MustCompile("^" + g.GitPathPrefix + sample + "$") } + globToRegexp := func(glob string) string { + r := glob + r = strings.ReplaceAll(r, ".", "\\.") + r = strings.ReplaceAll(r, "*", ".*") + return r + } + // Subdirectories of apps and prototypes are named after plugins plugins := []string{ g.ArgoCDDataDirName, @@ -86,17 +93,17 @@ func (g *Globe) runSmartMode(changedFiles ChangedFiles) EnvAppMap { "env": { e("(" + g.EnvironmentBaseDir + ".*)/" + g.EnvsDir + "/" + g.YttStepDirName + "/.*"), e("(" + g.EnvironmentBaseDir + ".*)/" + g.EnvsDir + "/" + g.ArgoCDDataDirName + "/.*"), - e("(" + g.EnvironmentBaseDir + ".*)/" + g.EnvironmentDataFileName), + e("(" + g.EnvironmentBaseDir + ".*)/" + globToRegexp(g.EnvironmentDataFileName)), }, // Prototype name is the only submatch "prototype": { e(g.PrototypesDir + "/(.+)/" + pluginsPattern + "/.*"), - e(g.PrototypesDir + "/(.+)/" + g.ApplicationDataFileName), + e(g.PrototypesDir + "/(.+)/" + globToRegexp(g.ApplicationDataFileName)), }, // Env path and app name are the submatches "app": { e("(" + g.EnvironmentBaseDir + ".*)/" + g.AppsDir + "/([^/]+)/" + pluginsPattern + "/.*"), - e("(" + g.EnvironmentBaseDir + ".*)/" + g.AppsDir + "/([^/]+)/" + g.ApplicationDataFileName), + e("(" + g.EnvironmentBaseDir + ".*)/" + g.AppsDir + "/([^/]+)/" + globToRegexp(g.ApplicationDataFileName)), }, } diff --git a/internal/myks/smart_mode_test.go b/internal/myks/smart_mode_test.go index 2018a32c..3510ecab 100644 --- a/internal/myks/smart_mode_test.go +++ b/internal/myks/smart_mode_test.go @@ -238,7 +238,7 @@ func TestGlobe_runSmartMode(t *testing.T) { }, { "change to app", - ChangedFiles{"envs/env1/_apps/app1/app-data.ytt.yaml": "M"}, + ChangedFiles{"envs/env1/_apps/app1/app-data.variable.yaml": "M"}, renderedEnvApps, EnvAppMap{ "envs/env1": {"app1"}, From cc5dd67f2f626283552482c9c34752308a879494 Mon Sep 17 00:00:00 2001 From: Kris Budde Date: Sun, 29 Dec 2024 22:02:39 +0100 Subject: [PATCH 3/3] make environmentdata files glob-able --- internal/myks/environment.go | 4 +--- internal/myks/globe.go | 27 +++++++++++++++++---------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/internal/myks/environment.go b/internal/myks/environment.go index a4d849fc..6ab1b6d3 100644 --- a/internal/myks/environment.go +++ b/internal/myks/environment.go @@ -34,9 +34,7 @@ type Environment struct { foundApplications map[string]string } -func NewEnvironment(g *Globe, dir string) (*Environment, error) { - envDataFile := filepath.Join(dir, g.EnvironmentDataFileName) - +func NewEnvironment(g *Globe, dir string, envDataFile string) (*Environment, error) { env := &Environment{ Dir: dir, EnvironmentDataFile: envDataFile, diff --git a/internal/myks/globe.go b/internal/myks/globe.go index 5a7cbc27..c6ccf6c5 100644 --- a/internal/myks/globe.go +++ b/internal/myks/globe.go @@ -46,9 +46,9 @@ type Globe struct { // Data values schema file name DataSchemaFileName string `default:"data-schema.ytt.yaml"` // Application data file name - ApplicationDataFileName string `default:"app-data.*.yaml"` + ApplicationDataFileName string `default:"app-data*.yaml"` // Environment data file name - EnvironmentDataFileName string `default:"env-data.ytt.yaml"` + EnvironmentDataFileName string `default:"env-data*.yaml"` // Rendered environment data file name RenderedEnvironmentDataFileName string `default:"env-data.yaml"` // Myks runtime data file name @@ -406,20 +406,27 @@ func (g *Globe) collectEnvironmentsInPath(searchPath string) []string { return err } if d != nil && d.IsDir() { - if ok, err := isExist(filepath.Join(path, g.EnvironmentDataFileName)); err != nil { + files, err := filepath.Glob(filepath.Join(path, g.EnvironmentDataFileName)) + if err != nil { return err - } else if ok { - env, err := NewEnvironment(g, path) + } + if len(files) == 0 { + return nil + } + // Try all files in the directory until a valid environment is found + for _, file := range files { + env, err := NewEnvironment(g, path, file) if err == nil { g.environments[path] = env result = append(result, path) - } else { - log.Debug(). - Err(err). - Str("path", path). - Msg("Unable to collect environment, might be base or parent environment. Skipping") + return nil } } + log.Debug(). + Str("path", path). + Strs("files", files). + Msg("Unable to collect environment, might be base or parent environment. Skipping") + } return nil })