cmd/scollector: enable SWbemServices worker for better WMI performance in Windows

cmd/scollector/collectors/dsc_windows.go

@@ -10,6 +10,7 @@ import (
 
 	"bosun.org/metadata"
 	"bosun.org/opentsdb"
+	"bosun.org/slog"
 	"bosun.org/util"
 	"github.com/StackExchange/mof"
 )
@@ -23,6 +24,9 @@ const (
 	dscLCM    = "dsc.lcm."
 	dscMof    = "dsc.mof."
 	dscStatus = "dsc.status."
+
+	//dscDefaultConfigurationName is used when the ConfigurationName value in the struct is NULL (non-pull based configuration usually)
+	dscDefaultConfigurationName = "__no_name__"
 )
 
 var (
@@ -83,17 +87,17 @@ func c_dsc_status() (opentsdb.MultiDataPoint, error) {
 		if err.Error() == "exit status 2147749889" {
 			return md, nil
 		} else {
-			return nil, err
+			return nil, slog.Wrap(err)
 		}
 	}
 	dscstatusbuffer := new(bytes.Buffer)
 	_, err = dscstatusbuffer.ReadFrom(dscstatusmof)
 	if err != nil {
-		return nil, err
+		return nil, slog.Wrap(err)
 	}
 	err = mof.Unmarshal(dscstatusbuffer.Bytes(), &dst)
 	if err != nil {
-		return nil, err
+		return nil, slog.Wrap(err)
 	}
 	if dst.ReturnValue != 0 {
 		return nil, fmt.Errorf("GetConfigurationStatus ReturnValue %v", dst.ReturnValue)
@@ -112,16 +116,18 @@ func c_dsc_status() (opentsdb.MultiDataPoint, error) {
 		Add(&md, dscStatus+"run_type", dscTypeToStatusCode(v.Type), nil, metadata.Gauge, metadata.Count, descWinDSCType)
 		configurations := make(map[string]dscResourceCount)
 		for _, r := range v.ResourcesInDesiredState {
-			c := configurations[r.ConfigurationName]
+			name := dscGetConfigurationName(r.ConfigurationName)
+			c := configurations[name]
 			c.Success++
 			c.Duration += r.DurationInSeconds
-			configurations[r.ConfigurationName] = c
+			configurations[name] = c
 		}
 		for _, r := range v.ResourcesNotInDesiredState {
-			c := configurations[r.ConfigurationName]
+			name := dscGetConfigurationName(r.ConfigurationName)
+			c := configurations[name]
 			c.Failed++
 			c.Duration += r.DurationInSeconds
-			configurations[r.ConfigurationName] = c
+			configurations[name] = c
 		}
 		for key, value := range configurations {
 			Add(&md, dscStatus+"resources", value.Success, opentsdb.TagSet{"state": "Success", "configuration": key}, metadata.Gauge, metadata.Count, descWinDSCResourceState)
@@ -247,3 +253,10 @@ func dscStartDateToAge(startdate string) float64 {
 	}
 	return time.Now().UTC().Sub(t).Seconds()
 }
+
+func dscGetConfigurationName(Name string) string {
+	if Name != "" {
+		return Name
+	}
+	return dscDefaultConfigurationName
+}

cmd/scollector/conf/conf.go

@@ -32,7 +32,8 @@ type Conf struct {
 	MaxMem uint64
 	// Filter filters collectors matching these terms.
 	Filter []string
-	// PProf is an IP:Port binding to be used for debugging with pprof package.
+	// PProf is an IP:Port binding to be used for debugging with pprof and expvar package.
+	// When enabled data is exposed via http://host:port/debug/pprof or /debug/vars
 	// Examples: localhost:6060 for loopback or :6060 for all IP addresses.
 	PProf string
 	// MetricFilters takes regular expressions and includes only indicies that
@@ -55,6 +56,9 @@ type Conf struct {
 	// SNMPTimeout is the number of seconds to wait for SNMP responses (default 30)
 	SNMPTimeout int
 
+	// UseSWbemServicesClient specifies if the wmi package should use SWbemServices.
+	UseSWbemServicesClient bool
+
 	HAProxy        []HAProxy
 	SNMP           []SNMP
 	MIBS           map[string]MIB

cmd/scollector/conf/conf_darwin.go

@@ -6,3 +6,5 @@ type ServiceParams struct {
 	Name      string
 	WatchProc bool
 }
+
+func (c *Conf) InitializeSWbemServices() {}

cmd/scollector/conf/conf_linux.go

@@ -11,3 +11,5 @@ type ServiceParams struct {
 	Name      string
 	WatchProc bool
 }
+
+func (c *Conf) InitializeSWbemServices() {}

cmd/scollector/conf/conf_windows.go

@@ -1,5 +1,10 @@
 package conf
 
+import (
+	"bosun.org/slog"
+	"github.com/StackExchange/wmi"
+)
+
 type ProcessParams struct {
 	Name string
 }
@@ -8,3 +13,12 @@ type ServiceParams struct {
 	Name      string
 	WatchProc bool
 }
+
+func (c *Conf) InitializeSWbemServices() {
+	slog.Infof("Initializing SWbemServices")
+	s, err := wmi.InitializeSWbemServices(wmi.DefaultClient)
+	if err != nil {
+		slog.Fatal(err)
+	}
+	wmi.DefaultClient.SWbemServicesClient = s
+}

cmd/scollector/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"encoding/json"
+	_ "expvar"
 	"flag"
 	"fmt"
 	"io/ioutil"
@@ -127,6 +128,9 @@ func main() {
 	if conf.SNMPTimeout > 0 {
 		snmp.Timeout = conf.SNMPTimeout
 	}
+	if conf.UseSWbemServicesClient {
+		conf.InitializeSWbemServices()
+	}
 	var err error
 	check := func(e error) {
 		if e != nil {

cmd/scollector/service_windows.go

@@ -16,6 +16,8 @@ import (
 
 var win_service_command = flag.String("winsvc", "", "For Windows Service, can be install, remove, start, stop")
 
+var serviceRunning = false
+
 func init() {
 	mains = append(mains, win_service_main)
 }
@@ -39,6 +41,15 @@ func win_service_main() {
 		}
 		if !isIntSess {
 			go runService(svcName, false)
+			for {
+				//Need to wait for service go routine to finish initializing. Otherwise the collector goroutines could
+				//use all the CPU and cause Windows Service API to bail with a service unresponsive on startup error.
+				//If service doesn't start within 30 seconds then the Windows Service API will kill the process.
+				time.Sleep(time.Millisecond * 200)
+				if serviceRunning {
+					break
+				}
+			}
 		}
 		return
 	default:
@@ -155,6 +166,7 @@ func (m *s) Execute(args []string, r <-chan svc.ChangeRequest, changes chan<- sv
 	const cmdsAccepted = svc.AcceptStop | svc.AcceptShutdown
 	changes <- svc.Status{State: svc.StartPending}
 	changes <- svc.Status{State: svc.Running, Accepts: cmdsAccepted}
+	serviceRunning = true
 loop:
 	for c := range r {
 		switch c.Cmd {