feat: add a Prometheus HTTP service discovery end-point

.github/workflows/go.yml

@@ -20,7 +20,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.15
+        go-version: 1.17
 
     - name: Build
       run: go build -v ./...
@@ -40,7 +40,7 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.15
+          go-version: 1.17
       - name: Install wwhrd
         env:
           GO111MODULE: 'off'

README.md

@@ -202,19 +202,19 @@ All pollers are defined in `harvest.yml`, the main configuration file of Harvest
 
 | parameter              | type         | description                                      | default                |
 |------------------------|--------------|--------------------------------------------------|------------------------|
-| Poller name (header)   | **required** |  poller name, user-defined value                 |                        |
-| `datacenter`           | **required** |  datacenter name, user-defined value                               |                        |
+| Poller name (header)   | **required** |  Poller name, user-defined value                 |                        |
+| `datacenter`           | **required** |  Datacenter name, user-defined value                               |                        |
 | `addr`                 | required by some collectors |  IPv4 or FQDN of the target system                     |                        |
-| `collectors`           | **required** | list of collectors to run for this poller |   |
-| `exporters`            | **required** | list of exporter names from the `Exporters` section. Note: this should be the name of the exporter (e.g. `prometheus1`), not the value of the `exporter` key (e.g. `Prometheus`)   |                   |
-| `auth_style`           | required by Zapi* collectors |  either `basic_auth` or `certificate_auth`  | `basic_auth` |
+| `collectors`           | **required** | List of collectors to run for this poller |   |
+| `exporters`            | **required** | List of exporter names from the `Exporters` section. Note: this should be the name of the exporter (e.g. `prometheus1`), not the value of the `exporter` key (e.g. `Prometheus`)   |                   |
+| `auth_style`           | required by Zapi* collectors |  Either `basic_auth` or `certificate_auth`  | `basic_auth` |
 | `username`, `password` | required if `auth_style` is `basic_auth` |  |              |
 | `ssl_cert`, `ssl_key`  | optional if `auth_style` is `certificate_auth` | Absolute paths to SSL (client) certificate and key used to authenticate with the target system.<br /><br />If not provided, the poller will look for `<hostname>.key` and `<hostname>.pem` in `$HARVEST_HOME/cert/`.<br/><br/>To create certificates for ONTAP systems, see [using certificate authentication](docs/AuthAndPermissions.md#using-certificate-authentication)                        |              |
 | `use_insecure_tls`     | optional, bool |  If true, disable TLS verification when connecting to ONTAP cluster  | false         |
-| `labels`               | optional, list of key-value pairs              | each of the key-value pairs will be added to a poller's metrics. Details [below](#labels)                                                                                                                                                                                                                                                                                 |                    |
+| `labels`               | optional, list of key-value pairs              | Each of the key-value pairs will be added to a poller's metrics. Details [below](#labels)                                                                                                                                                                                                                                                                                 |                    |
 | `log_max_bytes`        |  | Maximum size of the log file before it will be rotated | `10000000` (10 mb) |
 | `log_max_files`        |  | Number of rotated log files to keep | `10` |
-| `log`                  | optional, list of collector names              | matching collectors log their ZAPI request/response                                                                                                                                                                                                                                                                                                                         |                    |
+| `log`                  | optional, list of collector names              | Matching collectors log their ZAPI request/response                                                                                                                                                                                                                                                                                                                         |                    |
 
 ## Defaults
 This section is optional. If there are parameters identical for all your pollers (e.g. datacenter, authentication method, login preferences), they can be grouped under this section. The poller section will be checked first and if the values aren't found there, the defaults will be consulted.

cmd/admin/admin.go

@@ -0,0 +1,255 @@
+package admin
+
+import (
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/pkgerrors"
+	"github.com/spf13/cobra"
+	"github.com/zekroTJA/timedmap"
+	"goharvest2/pkg/conf"
+	"goharvest2/pkg/util"
+	"net/http"
+	"os"
+	"os/signal"
+	"strings"
+	"time"
+)
+
+type Admin struct {
+	listen           string
+	logger           zerolog.Logger
+	localIP          string
+	pollerToPromAddr *timedmap.TimedMap
+	httpSD           conf.Httpsd
+	updateCacheEvery time.Duration
+	expireAfter      time.Duration
+}
+
+func (a *Admin) startServer() {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/api/v1/sd", a.ApiSD)
+
+	a.logger.Debug().Str("listen", a.listen).Msg("Admin node starting")
+	server := &http.Server{
+		Addr:    a.listen,
+		Handler: mux,
+	}
+	if a.httpSD.TLS.KeyFile != "" {
+		server.TLSConfig = &tls.Config{
+			MinVersion: tls.VersionTLS13,
+		}
+	}
+
+	done := make(chan bool)
+	quit := make(chan os.Signal, 1)
+	signal.Notify(quit, os.Interrupt)
+
+	go func() {
+		<-quit
+		a.logger.Info().Msg("Admin node is shutting down")
+
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+
+		if err := server.Shutdown(ctx); err != nil {
+			a.logger.Fatal().
+				Err(err).
+				Msg("Could not gracefully shutdown the admin node")
+		}
+		close(done)
+	}()
+
+	a.logger.Info().
+		Str("listen", a.listen).
+		Bool("TLS", a.httpSD.TLS.KeyFile != "").
+		Bool("BasicAuth", a.httpSD.AuthBasic.Username != "").
+		Msg("Admin node started")
+
+	if a.httpSD.TLS.KeyFile != "" {
+		if err := server.ListenAndServeTLS(a.httpSD.TLS.CertFile, a.httpSD.TLS.KeyFile); err != nil && err != http.ErrServerClosed {
+			a.logger.Fatal().Err(err).
+				Str("listen", a.listen).
+				Str("ssl_cert", a.httpSD.TLS.CertFile).
+				Str("ssl_key", a.httpSD.TLS.KeyFile).
+				Msg("Admin node could not listen")
+		}
+	} else {
+		if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+			a.logger.Fatal().Err(err).
+				Str("listen", a.listen).
+				Msg("Admin node could not listen")
+		}
+	}
+
+	<-done
+	a.logger.Info().Msg("Admin node stopped")
+}
+
+func (a *Admin) ApiSD(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("content-type", "application/json")
+	if a.httpSD.AuthBasic.Username != "" {
+		user, pass, ok := r.BasicAuth()
+		if !ok || !a.verifyAuth(user, pass) {
+			w.Header().Set("WWW-Authenticate", `Basic realm="api"`)
+			http.Error(w, "Unauthorized", http.StatusUnauthorized)
+			return
+		}
+	}
+	if r.Method == "PUT" {
+		a.apiPublish(w, r)
+	} else if r.Method == "GET" {
+		w.WriteHeader(200)
+		_, _ = fmt.Fprintf(w, `[{"targets": [%s]}]`, a.makeTargets())
+	} else {
+		w.WriteHeader(400)
+	}
+}
+
+func (a *Admin) setupLogger() {
+	zerolog.SetGlobalLevel(zerolog.InfoLevel)
+	zerolog.ErrorStackMarshaler = pkgerrors.MarshalStack
+
+	a.logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).
+		With().Caller().Timestamp().Logger()
+}
+
+type pollerDetails struct {
+	Name string `json:"Name,omitempty"`
+	Ip   string `json:"Ip,omitempty"`
+	Port int    `json:"Port,omitempty"`
+}
+
+func (a *Admin) apiPublish(w http.ResponseWriter, r *http.Request) {
+	decoder := json.NewDecoder(r.Body)
+	var publish pollerDetails
+	err := decoder.Decode(&publish)
+	if err != nil {
+		a.logger.Err(err).Msg("Unable to parse publish json")
+		w.WriteHeader(400)
+		return
+	}
+	a.pollerToPromAddr.Set(publish.Name, publish, a.expireAfter)
+	a.logger.Debug().Str("name", publish.Name).Str("ip", publish.Ip).Int("port", publish.Port).
+		Msg("Published poller")
+	_, _ = fmt.Fprintf(w, "OK")
+}
+
+func (a *Admin) makeTargets() string {
+	targets := make([]string, 0)
+	for _, details := range a.pollerToPromAddr.Snapshot() {
+		pd := details.(pollerDetails)
+		targets = append(targets, fmt.Sprintf(`"%s:%d"`, pd.Ip, pd.Port))
+	}
+	a.logger.Debug().Int("size", len(targets)).Msg("makeTargets")
+	return strings.Join(targets, ",")
+}
+
+type tlsOptions struct {
+	DnsName   []string
+	Ipaddress []string
+	Days      int
+}
+
+var opts = &tlsOptions{}
+
+func Cmd() *cobra.Command {
+	admin := &cobra.Command{
+		Use:   "admin",
+		Short: "Harvest admin commands",
+	}
+	admin.AddCommand(&cobra.Command{
+		Use:   "start",
+		Short: "Start Harvest admin node",
+		Run:   doAdmin,
+	})
+	ctls := &cobra.Command{
+		Use:   "tls",
+		Short: "Builtin helpers for creating certificates",
+		Long:  "This command has subcommands for interacting with Harvest TLS.",
+	}
+	tlsCreate := &cobra.Command{
+		Use:   "create",
+		Short: "Create ",
+	}
+	tlsServer := &cobra.Command{
+		Use:   "server",
+		Short: "Create a new server certificates",
+		Run:   doTLS,
+	}
+	tlsCreate.AddCommand(tlsServer)
+	tlsCreate.PersistentFlags().StringSliceVar(
+		&opts.DnsName, "dnsname", []string{},
+		"Additional dns names for Subject Alternative Names. "+
+			"localhost is always included. Comma-separated list or provide flag multiple times",
+	)
+	tlsCreate.PersistentFlags().StringSliceVar(
+		&opts.Ipaddress, "ip", []string{},
+		"Additional IP addresses for Subject Alternative Names. "+
+			"127.0.0.1 is always included. Comma-separated list or provide flag multiple times",
+	)
+	tlsCreate.PersistentFlags().IntVarP(
+		&opts.Days, "days", "d", 365,
+		"Number of days the certificate is valid.",
+	)
+	ctls.AddCommand(tlsCreate)
+	admin.AddCommand(ctls)
+	return admin
+}
+
+func doTLS(_ *cobra.Command, _ []string) {
+	GenerateAdminCerts(opts, "admin")
+}
+
+func doAdmin(c *cobra.Command, _ []string) {
+	var configPath = c.Root().PersistentFlags().Lookup("config").Value.String()
+	err := conf.LoadHarvestConfig(configPath)
+	if err != nil {
+		return
+	}
+
+	a := newAdmin(configPath)
+	a.startServer()
+}
+
+func newAdmin(configPath string) Admin {
+	a := Admin{
+		httpSD: conf.Config.Admin.Httpsd,
+		listen: conf.Config.Admin.Httpsd.Listen,
+	}
+	a.setupLogger()
+	if a.listen == "" {
+		a.logger.Fatal().
+			Str("config", configPath).
+			Msg("Admin.address is empty in config. Must be a valid address")
+	}
+	if a.httpSD.TLS != (conf.TLS{}) {
+		util.CheckCert(a.httpSD.TLS.CertFile, "ssl_cert", configPath, a.logger)
+		util.CheckCert(a.httpSD.TLS.KeyFile, "ssl_key", configPath, a.logger)
+	}
+
+	a.localIP, _ = util.FindLocalIP()
+	a.expireAfter = a.setDuration(a.httpSD.ExpireAfter, 1*time.Minute, "expire_after")
+	a.pollerToPromAddr = timedmap.New(a.expireAfter)
+	a.logger.Debug().
+		Str("expireAfter", a.expireAfter.String()).
+		Str("localIP", a.localIP).
+		Msg("newAdmin")
+
+	return a
+}
+
+func (a *Admin) setDuration(every string, defaultDur time.Duration, name string) time.Duration {
+	if every == "" {
+		return defaultDur
+	}
+	everyDur, err := time.ParseDuration(every)
+	if err != nil {
+		a.logger.Warn().Err(err).Str(name, every).
+			Msgf("Failed to parse %s. Using %s", name, defaultDur.String())
+		everyDur = defaultDur
+	}
+	return everyDur
+}