From b105b8d58c26207588ec6e87429a73419ae25609 Mon Sep 17 00:00:00 2001 From: Chris Grindstaff Date: Fri, 29 Sep 2023 13:47:24 -0400 Subject: [PATCH] feat: Harvest should load templates from a set of conf directories --- cmd/tools/generate/generate.go | 98 ++++++++++++------- cmd/tools/generate/generate_test.go | 22 +++++ .../onePollerPerContainer/docker-compose.tmpl | 1 - 3 files changed, 86 insertions(+), 35 deletions(-) create mode 100644 cmd/tools/generate/generate_test.go diff --git a/cmd/tools/generate/generate.go b/cmd/tools/generate/generate.go index df69152e2..ad4c00448 100644 --- a/cmd/tools/generate/generate.go +++ b/cmd/tools/generate/generate.go @@ -29,7 +29,6 @@ type PollerInfo struct { ContainerName string ShowPorts bool IsFull bool - TemplateDir string CertDir string Mounts []string } @@ -61,11 +60,12 @@ type options struct { filesdPath string showPorts bool outputPath string - templateDir string certDir string promPort int grafanaPort int mounts []string + configPath string + confPath string } var opts = &options{ @@ -105,22 +105,28 @@ var metricCmd = &cobra.Command{ } func doDockerFull(cmd *cobra.Command, _ []string) { - var config = cmd.Root().PersistentFlags().Lookup("config") - generateDocker(conf.ConfigPath(config.Value.String()), full) + addRootOptions(cmd) + generateDocker(full) } + func doSystemd(cmd *cobra.Command, _ []string) { - var config = cmd.Root().PersistentFlags().Lookup("config") - generateSystemd(conf.ConfigPath(config.Value.String())) + addRootOptions(cmd) + generateSystemd() } func doDockerCompose(cmd *cobra.Command, _ []string) { - var config = cmd.Root().PersistentFlags().Lookup("config") - generateDocker(conf.ConfigPath(config.Value.String()), harvest) + addRootOptions(cmd) + generateDocker(harvest) } func doGenerateMetrics(cmd *cobra.Command, _ []string) { - var config = cmd.Root().PersistentFlags().Lookup("config") - generateMetrics(conf.ConfigPath(config.Value.String())) + addRootOptions(cmd) + generateMetrics() +} + +func addRootOptions(cmd *cobra.Command) { + opts.configPath = conf.ConfigPath(cmd.Root().PersistentFlags().Lookup("config").Value.String()) + opts.confPath = cmd.Root().PersistentFlags().Lookup("confpath").Value.String() } const ( @@ -134,15 +140,13 @@ func normalizeContainerNames(name string) string { return strings.ToLower(re.ReplaceAllString(name, "-")) } -func generateDocker(path string, kind int) { +func generateDocker(kind int) { var ( - pollerTemplate PollerTemplate - configFilePath string - templateDirPath string - certDirPath string - filesd []string - extraMounts []string - out *os.File + pollerTemplate PollerTemplate + configFilePath string + certDirPath string + filesd []string + out *os.File ) pollerTemplate = PollerTemplate{} @@ -150,19 +154,13 @@ func generateDocker(path string, kind int) { opts.grafanaPort, opts.promPort, } - _, err := conf.LoadHarvestConfig(path) + _, err := conf.LoadHarvestConfig(opts.configPath) if err != nil { logErrAndExit(err) } - configFilePath = asComposePath(path) - templateDirPath = asComposePath(opts.templateDir) + configFilePath = asComposePath(opts.configPath) certDirPath = asComposePath(opts.certDir) - extraMounts = make([]string, 0, len(opts.mounts)) - for _, mount := range opts.mounts { - extraMounts = append(extraMounts, asComposePath(mount)) - } - for _, v := range conf.Config.PollersOrdered { port, _ := conf.GetPrometheusExporterPorts(v, true) pollerInfo := PollerInfo{ @@ -175,9 +173,8 @@ func generateDocker(path string, kind int) { ContainerName: normalizeContainerNames("poller_" + v), ShowPorts: opts.showPorts, IsFull: kind == full, - TemplateDir: templateDirPath, CertDir: certDirPath, - Mounts: extraMounts, + Mounts: makeMounts(v), } pollerTemplate.Pollers = append(pollerTemplate.Pollers, pollerInfo) filesd = append(filesd, fmt.Sprintf("- targets: ['%s:%d']", pollerInfo.ServiceName, pollerInfo.Port)) @@ -272,6 +269,40 @@ func generateDocker(path string, kind int) { } } +// setup mount(s) for the confpath and any CLI-passed mounts +func makeMounts(pollerName string) []string { + var mounts = opts.mounts + + p, err := conf.PollerNamed(pollerName) + if err != nil { + logErrAndExit(err) + } + + confPath := opts.confPath + if confPath == "conf" { + confPath = p.ConfPath + } + + if confPath == "" { + mounts = append(mounts, toMount("./conf")) + } else { + paths := strings.Split(confPath, ":") + for _, path := range paths { + mounts = append(mounts, toMount(path)) + } + } + + return mounts +} + +func toMount(hostPath string) string { + hostPath = asComposePath(hostPath) + if strings.HasPrefix(hostPath, "./") { + return hostPath + ":" + "/opt/harvest/" + hostPath[2:] + } + return hostPath + ":" + hostPath +} + func copyFiles(srcPath, destPath string) error { filesToExclude := map[string]bool{ "harvest.yml": true, @@ -346,9 +377,9 @@ func silentClose(body io.ReadCloser) { _ = body.Close() } -func generateSystemd(path string) { +func generateSystemd() { var adminService string - _, err := conf.LoadHarvestConfig(path) + _, err := conf.LoadHarvestConfig(opts.configPath) if err != nil { logErrAndExit(err) } @@ -367,7 +398,7 @@ func generateSystemd(path string) { println("and " + color.Colorize("cp "+harvestAdminService+" /etc/systemd/system/", color.Green)) } println("and then run " + color.Colorize("systemctl daemon-reload", color.Green)) - writeAdminSystemd(path) + writeAdminSystemd(opts.configPath) // reorder list of pollers so that unix collectors are last, see https://github.com/NetApp/harvest/issues/643 pollers := make([]string, 0) unixPollers := make([]string, 0) @@ -422,7 +453,7 @@ func writeAdminSystemd(configFp string) { println(color.Colorize("✓", color.Green) + " HTTP SD file: " + harvestAdminService + " created") } -func generateMetrics(path string) { +func generateMetrics() { var ( poller *conf.Poller err error @@ -430,7 +461,7 @@ func generateMetrics(path string) { zapiClient *zapi.Client ) - _, err = conf.LoadHarvestConfig(path) + _, err = conf.LoadHarvestConfig(opts.configPath) if err != nil { logErrAndExit(err) } @@ -480,7 +511,6 @@ func init() { "logging level (0=trace, 1=debug, 2=info, 3=warning, 4=error, 5=critical)", ) dFlags.StringVar(&opts.image, "image", "ghcr.io/netapp/harvest:latest", "Harvest image. Use rahulguptajss/harvest:latest to pull from Docker Hub") - dFlags.StringVar(&opts.templateDir, "templatedir", "./conf", "Harvest template dir path") dFlags.StringVar(&opts.certDir, "certdir", "./cert", "Harvest certificate dir path") dFlags.StringVarP(&opts.outputPath, "output", "o", "", "Output file path. ") dFlags.BoolVarP(&opts.showPorts, "port", "p", true, "Expose poller ports to host machine") diff --git a/cmd/tools/generate/generate_test.go b/cmd/tools/generate/generate_test.go new file mode 100644 index 000000000..bde7879b1 --- /dev/null +++ b/cmd/tools/generate/generate_test.go @@ -0,0 +1,22 @@ +package generate + +import "testing" + +func Test_toMount(t *testing.T) { + tests := []struct { + name string + hostPath string + want string + }{ + {name: "dot prefix", hostPath: "./abc/d", want: "./abc/d:/opt/harvest/abc/d"}, + {name: "absolute", hostPath: "/x/y/z", want: "/x/y/z:/x/y/z"}, + {name: "cwd", hostPath: "abc/d", want: "./abc/d:/opt/harvest/abc/d"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := toMount(tt.hostPath); got != tt.want { + t.Errorf("toMount() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/container/onePollerPerContainer/docker-compose.tmpl b/container/onePollerPerContainer/docker-compose.tmpl index 1654461aa..9c564f948 100644 --- a/container/onePollerPerContainer/docker-compose.tmpl +++ b/container/onePollerPerContainer/docker-compose.tmpl @@ -26,7 +26,6 @@ services: command: '--poller {{ .PollerName }} {{if .Port }}--promPort {{ .Port }} {{ end }} {{- if ne .LogLevel 2 }}--loglevel {{ .LogLevel }} {{ end}}--config /opt/harvest.yml' volumes: - - {{ .TemplateDir }}:/opt/harvest/conf - {{ .CertDir }}:/opt/harvest/cert - {{ .ConfigFile }}:/opt/harvest.yml {{- range .Mounts}}