Skip to content

Commit

Permalink
Add histogram metrics for infoblox calls
Browse files Browse the repository at this point in the history 
Signed-off-by: Jirka Kremser <[email protected]>
  • Loading branch information
@jkremser
jkremser committed Dec 15, 2021
1 parent 85c5982 commit f509e71
Show file tree
Hide file tree
Showing 4 changed files with 111 additions and 22 deletions.
94 changes: 75 additions & 19 deletions controllers/providers/dns/infoblox.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import (

externaldns "sigs.k8s.io/external-dns/endpoint"

ibclient "github.com/infobloxopen/infoblox-go-client"
ibcl "github.com/infobloxopen/infoblox-go-client"
k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1"
"github.com/k8gb-io/k8gb/controllers/depresolver"
"github.com/k8gb-io/k8gb/controllers/providers/assistant"
Expand All @@ -47,7 +47,7 @@ func NewInfobloxDNS(config depresolver.Config, assistant assistant.Assistant, cl
}
}

func (p *InfobloxProvider) sanitizeDelegateZone(local, upstream []ibclient.NameServer) []ibclient.NameServer {
func (p *InfobloxProvider) sanitizeDelegateZone(local, upstream []ibcl.NameServer) []ibcl.NameServer {
// Drop own records for straight away update
// And ensure local entries are up to date
// And final list is sorted
Expand All @@ -70,14 +70,14 @@ func (p *InfobloxProvider) CreateZoneDelegationForExternalDNS(gslb *k8gbv1beta1.
m.InfobloxIncrementZoneUpdateError(gslb)
return err
}
var delegateTo []ibclient.NameServer
var delegateTo []ibcl.NameServer

for _, address := range addresses {
nameServer := ibclient.NameServer{Address: address, Name: p.config.GetClusterNSName()}
nameServer := ibcl.NameServer{Address: address, Name: p.config.GetClusterNSName()}
delegateTo = append(delegateTo, nameServer)
}

findZone, err := objMgr.GetZoneDelegated(p.config.DNSZone)
findZone, err := p.getZoneDelegated(objMgr, p.config.DNSZone)
if err != nil {
m.InfobloxIncrementZoneUpdateError(gslb)
return err
Expand Down Expand Up @@ -124,7 +124,7 @@ func (p *InfobloxProvider) CreateZoneDelegationForExternalDNS(gslb *k8gbv1beta1.
Str("DNS zone", p.config.DNSZone).
Str("server list", fmt.Sprintf("%v", currentList)).
Msg("Updating delegated zone with the server list")
_, err = objMgr.UpdateZoneDelegated(findZone.Ref, currentList)
_, err = p.updateZoneDelegated(objMgr, findZone.Ref, currentList)
if err != nil {
m.InfobloxIncrementZoneUpdateError(gslb)
return err
Expand All @@ -140,7 +140,7 @@ func (p *InfobloxProvider) CreateZoneDelegationForExternalDNS(gslb *k8gbv1beta1.
log.Debug().
Str("records", fmt.Sprintf("%v", delegateTo)).
Msg("Delegated records")
_, err = objMgr.CreateZoneDelegated(p.config.DNSZone, delegateTo)
_, err = p.createZoneDelegated(objMgr, p.config.DNSZone, delegateTo)
if err != nil {
m.InfobloxIncrementZoneUpdateError(gslb)
return err
Expand All @@ -158,7 +158,7 @@ func (p *InfobloxProvider) Finalize(gslb *k8gbv1beta1.Gslb) error {
if err != nil {
return err
}
findZone, err := objMgr.GetZoneDelegated(p.config.DNSZone)
findZone, err := p.getZoneDelegated(objMgr, p.config.DNSZone)
if err != nil {
return err
}
Expand All @@ -172,15 +172,15 @@ func (p *InfobloxProvider) Finalize(gslb *k8gbv1beta1.Gslb) error {
log.Info().
Str("DNS zone", p.config.DNSZone).
Msg("Deleting delegated zone")
_, err := objMgr.DeleteZoneDelegated(findZone.Ref)
_, err := p.deleteZoneDelegated(objMgr, findZone.Ref)
if err != nil {
return err
}
}
}

heartbeatTXTName := p.config.GetClusterHeartbeatFQDN(gslb.Name)
findTXT, err := objMgr.GetTXTRecord(heartbeatTXTName)
findTXT, err := p.getTXTRecord(objMgr, heartbeatTXTName)
if err != nil {
return err
}
Expand All @@ -190,7 +190,7 @@ func (p *InfobloxProvider) Finalize(gslb *k8gbv1beta1.Gslb) error {
log.Info().
Str("TXT records", heartbeatTXTName).
Msg("Deleting split brain TXT record")
_, err := objMgr.DeleteTXTRecord(findTXT.Ref)
_, err := p.deleteTXTRecord(objMgr, findTXT.Ref)
if err != nil {
return err
}
Expand All @@ -215,19 +215,19 @@ func (p *InfobloxProvider) String() string {
return "Infoblox"
}

func (p *InfobloxProvider) saveHeartbeatTXTRecord(objMgr *ibclient.ObjectManager, gslb *k8gbv1beta1.Gslb) (err error) {
var heartbeatTXTRecord *ibclient.RecordTXT
func (p *InfobloxProvider) saveHeartbeatTXTRecord(objMgr *ibcl.ObjectManager, gslb *k8gbv1beta1.Gslb) (err error) {
var heartbeatTXTRecord *ibcl.RecordTXT
edgeTimestamp := fmt.Sprint(time.Now().UTC().Format("2006-01-02T15:04:05"))
heartbeatTXTName := p.config.GetClusterHeartbeatFQDN(gslb.Name)
heartbeatTXTRecord, err = objMgr.GetTXTRecord(heartbeatTXTName)
heartbeatTXTRecord, err = p.getTXTRecord(objMgr, heartbeatTXTName)
if err != nil {
return
}
if heartbeatTXTRecord == nil {
log.Info().
Str("HeartbeatTXTName", heartbeatTXTName).
Msg("Creating split brain TXT record")
_, err = objMgr.CreateTXTRecord(heartbeatTXTName, edgeTimestamp, uint(gslb.Spec.Strategy.DNSTtlSeconds), "default")
_, err = p.createTXTRecord(objMgr, heartbeatTXTName, edgeTimestamp, uint(gslb.Spec.Strategy.DNSTtlSeconds))
if err != nil {
m.InfobloxIncrementHeartbeatError(gslb)
return
Expand All @@ -236,7 +236,7 @@ func (p *InfobloxProvider) saveHeartbeatTXTRecord(objMgr *ibclient.ObjectManager
log.Info().
Str("HeartbeatTXTName", heartbeatTXTName).
Msg("Updating split brain TXT record")
_, err = objMgr.UpdateTXTRecord(heartbeatTXTName, edgeTimestamp)
_, err = p.updateTXTRecord(objMgr, heartbeatTXTName, edgeTimestamp)
if err != nil {
m.InfobloxIncrementHeartbeatError(gslb)
return
Expand All @@ -246,16 +246,16 @@ func (p *InfobloxProvider) saveHeartbeatTXTRecord(objMgr *ibclient.ObjectManager
return
}

func (p *InfobloxProvider) checkZoneDelegated(findZone *ibclient.ZoneDelegated) error {
func (p *InfobloxProvider) checkZoneDelegated(findZone *ibcl.ZoneDelegated) error {
if findZone.Fqdn != p.config.DNSZone {
err := fmt.Errorf("delegated zone returned from infoblox(%s) does not match requested gslb zone(%s)", findZone.Fqdn, p.config.DNSZone)
return err
}
return nil
}

func (p *InfobloxProvider) filterOutDelegateTo(delegateTo []ibclient.NameServer, fqdn string) (result []ibclient.NameServer) {
result = make([]ibclient.NameServer, 0)
func (p *InfobloxProvider) filterOutDelegateTo(delegateTo []ibcl.NameServer, fqdn string) (result []ibcl.NameServer) {
result = make([]ibcl.NameServer, 0)

for _, v := range delegateTo {
if v.Name != fqdn {
Expand All @@ -264,3 +264,59 @@ func (p *InfobloxProvider) filterOutDelegateTo(delegateTo []ibclient.NameServer,
}
return
}

func (p *InfobloxProvider) createZoneDelegated(o *ibcl.ObjectManager, fqdn string, d []ibcl.NameServer) (res *ibcl.ZoneDelegated, err error) {
start := time.Now()
res, err = o.CreateZoneDelegated(fqdn, d)
m.InfobloxObserveRequestDuration(start, metrics.CreateZoneDelegated, err == nil)
return
}

func (p *InfobloxProvider) getZoneDelegated(o *ibcl.ObjectManager, fqdn string) (res *ibcl.ZoneDelegated, err error) {
start := time.Now()
res, err = o.GetZoneDelegated(fqdn)
m.InfobloxObserveRequestDuration(start, metrics.GetZoneDelegated, err == nil)
return
}

func (p *InfobloxProvider) updateZoneDelegated(o *ibcl.ObjectManager, fqdn string, d []ibcl.NameServer) (res *ibcl.ZoneDelegated, err error) {
start := time.Now()
res, err = o.UpdateZoneDelegated(fqdn, d)
m.InfobloxObserveRequestDuration(start, metrics.UpdateZoneDelegated, err == nil)
return
}

func (p *InfobloxProvider) deleteZoneDelegated(o *ibcl.ObjectManager, fqdn string) (res string, err error) {
start := time.Now()
res, err = o.DeleteZoneDelegated(fqdn)
m.InfobloxObserveRequestDuration(start, metrics.DeleteZoneDelegated, err == nil)
return
}

func (p *InfobloxProvider) createTXTRecord(o *ibcl.ObjectManager, name string, text string, ttl uint) (res *ibcl.RecordTXT, err error) {
start := time.Now()
res, err = o.CreateTXTRecord(name, text, ttl, "default")
m.InfobloxObserveRequestDuration(start, metrics.CreateTXTRecord, err == nil)
return
}

func (p *InfobloxProvider) getTXTRecord(o *ibcl.ObjectManager, name string) (res *ibcl.RecordTXT, err error) {
start := time.Now()
res, err = o.GetTXTRecord(name)
m.InfobloxObserveRequestDuration(start, metrics.GetTXTRecord, err == nil)
return
}

func (p *InfobloxProvider) updateTXTRecord(o *ibcl.ObjectManager, name string, text string) (res *ibcl.RecordTXT, err error) {
start := time.Now()
res, err = o.UpdateTXTRecord(name, text)
m.InfobloxObserveRequestDuration(start, metrics.UpdateTXTRecord, err == nil)
return
}

func (p *InfobloxProvider) deleteTXTRecord(o *ibcl.ObjectManager, name string) (res string, err error) {
start := time.Now()
res, err = o.DeleteTXTRecord(name)
m.InfobloxObserveRequestDuration(start, metrics.DeleteTXTRecord, err == nil)
return
}
33 changes: 33 additions & 0 deletions controllers/providers/metrics/prometheus.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"runtime"
"strings"
"sync"
"time"

externaldns "sigs.k8s.io/external-dns/endpoint"

Expand All @@ -49,6 +50,7 @@ const (
K8gbGslbStatusCountForGeoIP = "k8gb_gslb_status_count_for_geoip"
K8gbInfobloxHeartbeatsTotal = "k8gb_infoblox_heartbeats_total"
K8gbInfobloxHeartbeatErrorsTotal = "k8gb_infoblox_heartbeat_errors_total"
K8gbInfobloxRequestDuration = "k8gb_infoblox_request_duration"
K8gbInfobloxZoneUpdatesTotal = "k8gb_infoblox_zone_updates_total"
K8gbInfobloxZoneUpdateErrorsTotal = "k8gb_infoblox_zone_update_errors_total"
K8gbEndpointStatusNum = "k8gb_endpoint_status_num"
Expand All @@ -64,6 +66,7 @@ type collectors struct {
K8gbGslbStatusCountForGeoip *prometheus.GaugeVec
K8gbGslbErrorsTotal *prometheus.CounterVec
K8gbGslbReconciliationLoopsTotal *prometheus.CounterVec
K8gbInfobloxRequestDuration *prometheus.HistogramVec
K8gbInfobloxZoneUpdatesTotal *prometheus.CounterVec
K8gbInfobloxZoneUpdateErrorsTotal *prometheus.CounterVec
K8gbInfobloxHeartbeatsTotal *prometheus.CounterVec
Expand All @@ -78,6 +81,21 @@ type PrometheusMetrics struct {
metrics collectors
}

// DNSProviderRequest is a label for histogram metric
type DNSProviderRequest string

const (
CreateZoneDelegated DNSProviderRequest = "ZoneCreate"
GetZoneDelegated DNSProviderRequest = "ZoneRead"
UpdateZoneDelegated DNSProviderRequest = "ZoneUpdate"
DeleteZoneDelegated DNSProviderRequest = "ZoneDelete"

CreateTXTRecord = "TXTRecordCreate"
GetTXTRecord = "TXTRecordRead"
UpdateTXTRecord = "TXTRecordUpdate"
DeleteTXTRecord = "TXTRecordDelete"
)

var regex = regexp.MustCompile("[A-Z]")

// newPrometheusMetrics creates new prometheus metrics instance
Expand Down Expand Up @@ -163,6 +181,12 @@ func (m *PrometheusMetrics) InfobloxIncrementHeartbeatError(gslb *k8gbv1beta1.Gs
m.metrics.K8gbInfobloxHeartbeatErrorsTotal.With(prometheus.Labels{"namespace": gslb.Namespace, "name": gslb.Name}).Inc()
}

func (m *PrometheusMetrics) InfobloxObserveRequestDuration(start time.Time, request DNSProviderRequest, success bool) {
duration := time.Since(start).Seconds()
fmt.Println("%v %s", duration, request)
m.metrics.K8gbInfobloxRequestDuration.With(prometheus.Labels{"request": string(request), "success": fmt.Sprintf("%t", success)}).Observe(duration)
}

func (m *PrometheusMetrics) SetRuntimeInfo(version, commit string) {
firstN := func(value string, n int) string {
if len(value) < n {
Expand Down Expand Up @@ -271,6 +295,15 @@ func (m *PrometheusMetrics) init() {
},
[]string{"namespace", "name", "status"},
)
m.metrics.K8gbInfobloxRequestDuration = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: K8gbInfobloxRequestDuration,
Help: "How long it took for Infoblox requests to complete, partitioned by request type. Round-trip time of http communication is included.",
Buckets: prometheus.ExponentialBuckets(.2, 4, 5),
},
[]string{"request", "success"},
)

m.metrics.K8gbInfobloxZoneUpdatesTotal = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: K8gbInfobloxZoneUpdatesTotal,
Expand Down
2 changes: 1 addition & 1 deletion controllers/providers/metrics/prometheus_result.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ type MetricResult struct {
value prometheus.Collector
}

// Get gets actual copy of metric defined by it's name
// Get gets actual copy of metric defined by its name
func (m *PrometheusMetrics) Get(name string) (r *MetricResult) {
return &MetricResult{value: m.registry()[name]}
}
Expand Down
4 changes: 2 additions & 2 deletions controllers/providers/metrics/prometheus_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,8 @@ func TestPrometheusRegistry(t *testing.T) {
items := []string{K8gbGslbErrorsTotal, K8gbGslbHealthyRecords, K8gbGslbReconciliationLoopsTotal,
K8gbGslbServiceStatusNum, K8gbGslbStatusCountForFailover, K8gbGslbStatusCountForRoundrobin,
K8gbGslbStatusCountForGeoIP, K8gbInfobloxHeartbeatsTotal, K8gbInfobloxHeartbeatErrorsTotal,
K8gbInfobloxZoneUpdatesTotal, K8gbInfobloxZoneUpdateErrorsTotal, K8gbEndpointStatusNum,
K8gbRuntimeInfo}
K8gbInfobloxRequestDuration, K8gbInfobloxZoneUpdatesTotal, K8gbInfobloxZoneUpdateErrorsTotal,
K8gbEndpointStatusNum, K8gbRuntimeInfo}
// act
registry := m.registry()
// assert
Expand Down

0 comments on commit f509e71

Please sign in to comment.