feat: Harvest should load templates from a set of conf directories

cmd/tools/generate/generate.go

@@ -29,7 +29,6 @@ type PollerInfo struct {
 	ContainerName string
 	ShowPorts     bool
 	IsFull        bool
-	TemplateDir   string
 	CertDir       string
 	Mounts        []string
 }
@@ -61,11 +60,12 @@ type options struct {
 	filesdPath  string
 	showPorts   bool
 	outputPath  string
-	templateDir string
 	certDir     string
 	promPort    int
 	grafanaPort int
 	mounts      []string
+	configPath  string
+	confPath    string
 }
 
 var opts = &options{
@@ -105,22 +105,28 @@ var metricCmd = &cobra.Command{
 }
 
 func doDockerFull(cmd *cobra.Command, _ []string) {
-	var config = cmd.Root().PersistentFlags().Lookup("config")
-	generateDocker(conf.ConfigPath(config.Value.String()), full)
+	addRootOptions(cmd)
+	generateDocker(full)
 }
+
 func doSystemd(cmd *cobra.Command, _ []string) {
-	var config = cmd.Root().PersistentFlags().Lookup("config")
-	generateSystemd(conf.ConfigPath(config.Value.String()))
+	addRootOptions(cmd)
+	generateSystemd()
 }
 
 func doDockerCompose(cmd *cobra.Command, _ []string) {
-	var config = cmd.Root().PersistentFlags().Lookup("config")
-	generateDocker(conf.ConfigPath(config.Value.String()), harvest)
+	addRootOptions(cmd)
+	generateDocker(harvest)
 }
 
 func doGenerateMetrics(cmd *cobra.Command, _ []string) {
-	var config = cmd.Root().PersistentFlags().Lookup("config")
-	generateMetrics(conf.ConfigPath(config.Value.String()))
+	addRootOptions(cmd)
+	generateMetrics()
+}
+
+func addRootOptions(cmd *cobra.Command) {
+	opts.configPath = conf.ConfigPath(cmd.Root().PersistentFlags().Lookup("config").Value.String())
+	opts.confPath = cmd.Root().PersistentFlags().Lookup("confpath").Value.String()
 }
 
 const (
@@ -134,35 +140,27 @@ func normalizeContainerNames(name string) string {
 	return strings.ToLower(re.ReplaceAllString(name, "-"))
 }
 
-func generateDocker(path string, kind int) {
+func generateDocker(kind int) {
 	var (
-		pollerTemplate  PollerTemplate
-		configFilePath  string
-		templateDirPath string
-		certDirPath     string
-		filesd          []string
-		extraMounts     []string
-		out             *os.File
+		pollerTemplate PollerTemplate
+		configFilePath string
+		certDirPath    string
+		filesd         []string
+		out            *os.File
 	)
 
 	pollerTemplate = PollerTemplate{}
 	promTemplate := PromTemplate{
 		opts.grafanaPort,
 		opts.promPort,
 	}
-	_, err := conf.LoadHarvestConfig(path)
+	_, err := conf.LoadHarvestConfig(opts.configPath)
 	if err != nil {
 		logErrAndExit(err)
 	}
-	configFilePath = asComposePath(path)
-	templateDirPath = asComposePath(opts.templateDir)
+	configFilePath = asComposePath(opts.configPath)
 	certDirPath = asComposePath(opts.certDir)
 
-	extraMounts = make([]string, 0, len(opts.mounts))
-	for _, mount := range opts.mounts {
-		extraMounts = append(extraMounts, asComposePath(mount))
-	}
-
 	for _, v := range conf.Config.PollersOrdered {
 		port, _ := conf.GetPrometheusExporterPorts(v, true)
 		pollerInfo := PollerInfo{
@@ -175,9 +173,8 @@ func generateDocker(path string, kind int) {
 			ContainerName: normalizeContainerNames("poller_" + v),
 			ShowPorts:     opts.showPorts,
 			IsFull:        kind == full,
-			TemplateDir:   templateDirPath,
 			CertDir:       certDirPath,
-			Mounts:        extraMounts,
+			Mounts:        makeMounts(v),
 		}
 		pollerTemplate.Pollers = append(pollerTemplate.Pollers, pollerInfo)
 		filesd = append(filesd, fmt.Sprintf("- targets: ['%s:%d']", pollerInfo.ServiceName, pollerInfo.Port))
@@ -272,6 +269,40 @@ func generateDocker(path string, kind int) {
 	}
 }
 
+// setup mount(s) for the confpath and any CLI-passed mounts
+func makeMounts(pollerName string) []string {
+	var mounts = opts.mounts
+
+	p, err := conf.PollerNamed(pollerName)
+	if err != nil {
+		logErrAndExit(err)
+	}
+
+	confPath := opts.confPath
+	if confPath == "conf" {
+		confPath = p.ConfPath
+	}
+
+	if confPath == "" {
+		mounts = append(mounts, toMount("./conf"))
+	} else {
+		paths := strings.Split(confPath, ":")
+		for _, path := range paths {
+			mounts = append(mounts, toMount(path))
+		}
+	}
+
+	return mounts
+}
+
+func toMount(hostPath string) string {
+	hostPath = asComposePath(hostPath)
+	if strings.HasPrefix(hostPath, "./") {
+		return hostPath + ":" + "/opt/harvest/" + hostPath[2:]
+	}
+	return hostPath + ":" + hostPath
+}
+
 func copyFiles(srcPath, destPath string) error {
 	filesToExclude := map[string]bool{
 		"harvest.yml":         true,
@@ -346,9 +377,9 @@ func silentClose(body io.ReadCloser) {
 	_ = body.Close()
 }
 
-func generateSystemd(path string) {
+func generateSystemd() {
 	var adminService string
-	_, err := conf.LoadHarvestConfig(path)
+	_, err := conf.LoadHarvestConfig(opts.configPath)
 	if err != nil {
 		logErrAndExit(err)
 	}
@@ -367,7 +398,7 @@ func generateSystemd(path string) {
 		println("and " + color.Colorize("cp "+harvestAdminService+" /etc/systemd/system/", color.Green))
 	}
 	println("and then run " + color.Colorize("systemctl daemon-reload", color.Green))
-	writeAdminSystemd(path)
+	writeAdminSystemd(opts.configPath)
 	// reorder list of pollers so that unix collectors are last, see https://github.com/NetApp/harvest/issues/643
 	pollers := make([]string, 0)
 	unixPollers := make([]string, 0)
@@ -422,15 +453,15 @@ func writeAdminSystemd(configFp string) {
 	println(color.Colorize("✓", color.Green) + " HTTP SD file: " + harvestAdminService + " created")
 }
 
-func generateMetrics(path string) {
+func generateMetrics() {
 	var (
 		poller     *conf.Poller
 		err        error
 		restClient *rest.Client
 		zapiClient *zapi.Client
 	)
 
-	_, err = conf.LoadHarvestConfig(path)
+	_, err = conf.LoadHarvestConfig(opts.configPath)
 	if err != nil {
 		logErrAndExit(err)
 	}
@@ -480,7 +511,6 @@ func init() {
 		"logging level (0=trace, 1=debug, 2=info, 3=warning, 4=error, 5=critical)",
 	)
 	dFlags.StringVar(&opts.image, "image", "ghcr.io/netapp/harvest:latest", "Harvest image. Use rahulguptajss/harvest:latest to pull from Docker Hub")
-	dFlags.StringVar(&opts.templateDir, "templatedir", "./conf", "Harvest template dir path")
 	dFlags.StringVar(&opts.certDir, "certdir", "./cert", "Harvest certificate dir path")
 	dFlags.StringVarP(&opts.outputPath, "output", "o", "", "Output file path. ")
 	dFlags.BoolVarP(&opts.showPorts, "port", "p", true, "Expose poller ports to host machine")

cmd/tools/generate/generate_test.go

@@ -0,0 +1,22 @@
+package generate
+
+import "testing"
+
+func Test_toMount(t *testing.T) {
+	tests := []struct {
+		name     string
+		hostPath string
+		want     string
+	}{
+		{name: "dot prefix", hostPath: "./abc/d", want: "./abc/d:/opt/harvest/abc/d"},
+		{name: "absolute", hostPath: "/x/y/z", want: "/x/y/z:/x/y/z"},
+		{name: "cwd", hostPath: "abc/d", want: "./abc/d:/opt/harvest/abc/d"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := toMount(tt.hostPath); got != tt.want {
+				t.Errorf("toMount() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

container/onePollerPerContainer/docker-compose.tmpl

@@ -26,7 +26,6 @@ services:
     command: '--poller {{ .PollerName }} {{if .Port }}--promPort {{ .Port }} {{ end }}
     {{- if ne .LogLevel 2 }}--loglevel {{ .LogLevel }} {{ end}}--config /opt/harvest.yml'
     volumes:
-      - {{ .TemplateDir }}:/opt/harvest/conf
       - {{ .CertDir }}:/opt/harvest/cert
       - {{ .ConfigFile }}:/opt/harvest.yml
     {{- range .Mounts}}