Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add histogram metrics for infoblox calls #805

Merged
merged 1 commit into from
Dec 16, 2021
Merged

Add histogram metrics for infoblox calls #805

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 75 additions & 19 deletions controllers/providers/dns/infoblox.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import (

externaldns "sigs.k8s.io/external-dns/endpoint"

ibclient "github.com/infobloxopen/infoblox-go-client"
ibcl "github.com/infobloxopen/infoblox-go-client"
k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1"
"github.com/k8gb-io/k8gb/controllers/depresolver"
"github.com/k8gb-io/k8gb/controllers/providers/assistant"
Expand All @@ -47,7 +47,7 @@ func NewInfobloxDNS(config depresolver.Config, assistant assistant.Assistant, cl
}
}

func (p *InfobloxProvider) sanitizeDelegateZone(local, upstream []ibclient.NameServer) []ibclient.NameServer {
func (p *InfobloxProvider) sanitizeDelegateZone(local, upstream []ibcl.NameServer) []ibcl.NameServer {
// Drop own records for straight away update
// And ensure local entries are up to date
// And final list is sorted
Expand All @@ -70,14 +70,14 @@ func (p *InfobloxProvider) CreateZoneDelegationForExternalDNS(gslb *k8gbv1beta1.
m.InfobloxIncrementZoneUpdateError(gslb)
return err
}
var delegateTo []ibclient.NameServer
var delegateTo []ibcl.NameServer

for _, address := range addresses {
nameServer := ibclient.NameServer{Address: address, Name: p.config.GetClusterNSName()}
nameServer := ibcl.NameServer{Address: address, Name: p.config.GetClusterNSName()}
delegateTo = append(delegateTo, nameServer)
}

findZone, err := objMgr.GetZoneDelegated(p.config.DNSZone)
findZone, err := p.getZoneDelegated(objMgr, p.config.DNSZone)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this change related to ibclient version?

Copy link
Member Author

@jkremser jkremser Dec 16, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nope, there is a new private function w/ almost the same signature that makes the actual call (objMgr.GetZoneDelegated) but this new function surrounds the call w/ time measurement so that we measure the request times in the histogram.

if err != nil {
m.InfobloxIncrementZoneUpdateError(gslb)
return err
Expand Down Expand Up @@ -124,7 +124,7 @@ func (p *InfobloxProvider) CreateZoneDelegationForExternalDNS(gslb *k8gbv1beta1.
Str("DNS zone", p.config.DNSZone).
Str("server list", fmt.Sprintf("%v", currentList)).
Msg("Updating delegated zone with the server list")
_, err = objMgr.UpdateZoneDelegated(findZone.Ref, currentList)
_, err = p.updateZoneDelegated(objMgr, findZone.Ref, currentList)
if err != nil {
m.InfobloxIncrementZoneUpdateError(gslb)
return err
Expand All @@ -140,7 +140,7 @@ func (p *InfobloxProvider) CreateZoneDelegationForExternalDNS(gslb *k8gbv1beta1.
log.Debug().
Str("records", fmt.Sprintf("%v", delegateTo)).
Msg("Delegated records")
_, err = objMgr.CreateZoneDelegated(p.config.DNSZone, delegateTo)
_, err = p.createZoneDelegated(objMgr, p.config.DNSZone, delegateTo)
if err != nil {
m.InfobloxIncrementZoneUpdateError(gslb)
return err
Expand All @@ -158,7 +158,7 @@ func (p *InfobloxProvider) Finalize(gslb *k8gbv1beta1.Gslb) error {
if err != nil {
return err
}
findZone, err := objMgr.GetZoneDelegated(p.config.DNSZone)
findZone, err := p.getZoneDelegated(objMgr, p.config.DNSZone)
if err != nil {
return err
}
Expand All @@ -172,15 +172,15 @@ func (p *InfobloxProvider) Finalize(gslb *k8gbv1beta1.Gslb) error {
log.Info().
Str("DNS zone", p.config.DNSZone).
Msg("Deleting delegated zone")
_, err := objMgr.DeleteZoneDelegated(findZone.Ref)
_, err := p.deleteZoneDelegated(objMgr, findZone.Ref)
if err != nil {
return err
}
}
}

heartbeatTXTName := p.config.GetClusterHeartbeatFQDN(gslb.Name)
findTXT, err := objMgr.GetTXTRecord(heartbeatTXTName)
findTXT, err := p.getTXTRecord(objMgr, heartbeatTXTName)
if err != nil {
return err
}
Expand All @@ -190,7 +190,7 @@ func (p *InfobloxProvider) Finalize(gslb *k8gbv1beta1.Gslb) error {
log.Info().
Str("TXT records", heartbeatTXTName).
Msg("Deleting split brain TXT record")
_, err := objMgr.DeleteTXTRecord(findTXT.Ref)
_, err := p.deleteTXTRecord(objMgr, findTXT.Ref)
if err != nil {
return err
}
Expand All @@ -215,19 +215,19 @@ func (p *InfobloxProvider) String() string {
return "Infoblox"
}

func (p *InfobloxProvider) saveHeartbeatTXTRecord(objMgr *ibclient.ObjectManager, gslb *k8gbv1beta1.Gslb) (err error) {
var heartbeatTXTRecord *ibclient.RecordTXT
func (p *InfobloxProvider) saveHeartbeatTXTRecord(objMgr *ibcl.ObjectManager, gslb *k8gbv1beta1.Gslb) (err error) {
var heartbeatTXTRecord *ibcl.RecordTXT
edgeTimestamp := fmt.Sprint(time.Now().UTC().Format("2006-01-02T15:04:05"))
heartbeatTXTName := p.config.GetClusterHeartbeatFQDN(gslb.Name)
heartbeatTXTRecord, err = objMgr.GetTXTRecord(heartbeatTXTName)
heartbeatTXTRecord, err = p.getTXTRecord(objMgr, heartbeatTXTName)
if err != nil {
return
}
if heartbeatTXTRecord == nil {
log.Info().
Str("HeartbeatTXTName", heartbeatTXTName).
Msg("Creating split brain TXT record")
_, err = objMgr.CreateTXTRecord(heartbeatTXTName, edgeTimestamp, uint(gslb.Spec.Strategy.DNSTtlSeconds), "default")
_, err = p.createTXTRecord(objMgr, heartbeatTXTName, edgeTimestamp, uint(gslb.Spec.Strategy.DNSTtlSeconds))
if err != nil {
m.InfobloxIncrementHeartbeatError(gslb)
return
Expand All @@ -236,7 +236,7 @@ func (p *InfobloxProvider) saveHeartbeatTXTRecord(objMgr *ibclient.ObjectManager
log.Info().
Str("HeartbeatTXTName", heartbeatTXTName).
Msg("Updating split brain TXT record")
_, err = objMgr.UpdateTXTRecord(heartbeatTXTName, edgeTimestamp)
_, err = p.updateTXTRecord(objMgr, heartbeatTXTName, edgeTimestamp)
if err != nil {
m.InfobloxIncrementHeartbeatError(gslb)
return
Expand All @@ -246,16 +246,16 @@ func (p *InfobloxProvider) saveHeartbeatTXTRecord(objMgr *ibclient.ObjectManager
return
}

func (p *InfobloxProvider) checkZoneDelegated(findZone *ibclient.ZoneDelegated) error {
func (p *InfobloxProvider) checkZoneDelegated(findZone *ibcl.ZoneDelegated) error {
if findZone.Fqdn != p.config.DNSZone {
err := fmt.Errorf("delegated zone returned from infoblox(%s) does not match requested gslb zone(%s)", findZone.Fqdn, p.config.DNSZone)
return err
}
return nil
}

func (p *InfobloxProvider) filterOutDelegateTo(delegateTo []ibclient.NameServer, fqdn string) (result []ibclient.NameServer) {
result = make([]ibclient.NameServer, 0)
func (p *InfobloxProvider) filterOutDelegateTo(delegateTo []ibcl.NameServer, fqdn string) (result []ibcl.NameServer) {
result = make([]ibcl.NameServer, 0)

for _, v := range delegateTo {
if v.Name != fqdn {
Expand All @@ -264,3 +264,59 @@ func (p *InfobloxProvider) filterOutDelegateTo(delegateTo []ibclient.NameServer,
}
return
}

func (p *InfobloxProvider) createZoneDelegated(o *ibcl.ObjectManager, fqdn string, d []ibcl.NameServer) (res *ibcl.ZoneDelegated, err error) {
start := time.Now()
res, err = o.CreateZoneDelegated(fqdn, d)
m.InfobloxObserveRequestDuration(start, metrics.CreateZoneDelegated, err == nil)
return
}

func (p *InfobloxProvider) getZoneDelegated(o *ibcl.ObjectManager, fqdn string) (res *ibcl.ZoneDelegated, err error) {
start := time.Now()
res, err = o.GetZoneDelegated(fqdn)
m.InfobloxObserveRequestDuration(start, metrics.GetZoneDelegated, err == nil)
return
}

func (p *InfobloxProvider) updateZoneDelegated(o *ibcl.ObjectManager, fqdn string, d []ibcl.NameServer) (res *ibcl.ZoneDelegated, err error) {
start := time.Now()
res, err = o.UpdateZoneDelegated(fqdn, d)
m.InfobloxObserveRequestDuration(start, metrics.UpdateZoneDelegated, err == nil)
return
}

func (p *InfobloxProvider) deleteZoneDelegated(o *ibcl.ObjectManager, fqdn string) (res string, err error) {
start := time.Now()
res, err = o.DeleteZoneDelegated(fqdn)
m.InfobloxObserveRequestDuration(start, metrics.DeleteZoneDelegated, err == nil)
return
}

func (p *InfobloxProvider) createTXTRecord(o *ibcl.ObjectManager, name string, text string, ttl uint) (res *ibcl.RecordTXT, err error) {
start := time.Now()
res, err = o.CreateTXTRecord(name, text, ttl, "default")
m.InfobloxObserveRequestDuration(start, metrics.CreateTXTRecord, err == nil)
return
}

func (p *InfobloxProvider) getTXTRecord(o *ibcl.ObjectManager, name string) (res *ibcl.RecordTXT, err error) {
start := time.Now()
res, err = o.GetTXTRecord(name)
m.InfobloxObserveRequestDuration(start, metrics.GetTXTRecord, err == nil)
return
}

func (p *InfobloxProvider) updateTXTRecord(o *ibcl.ObjectManager, name string, text string) (res *ibcl.RecordTXT, err error) {
start := time.Now()
res, err = o.UpdateTXTRecord(name, text)
m.InfobloxObserveRequestDuration(start, metrics.UpdateTXTRecord, err == nil)
return
}

func (p *InfobloxProvider) deleteTXTRecord(o *ibcl.ObjectManager, name string) (res string, err error) {
start := time.Now()
res, err = o.DeleteTXTRecord(name)
m.InfobloxObserveRequestDuration(start, metrics.DeleteTXTRecord, err == nil)
return
}
32 changes: 32 additions & 0 deletions controllers/providers/metrics/prometheus.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"runtime"
"strings"
"sync"
"time"

externaldns "sigs.k8s.io/external-dns/endpoint"

Expand All @@ -49,6 +50,7 @@ const (
K8gbGslbStatusCountForGeoIP = "k8gb_gslb_status_count_for_geoip"
K8gbInfobloxHeartbeatsTotal = "k8gb_infoblox_heartbeats_total"
K8gbInfobloxHeartbeatErrorsTotal = "k8gb_infoblox_heartbeat_errors_total"
K8gbInfobloxRequestDuration = "k8gb_infoblox_request_duration"
K8gbInfobloxZoneUpdatesTotal = "k8gb_infoblox_zone_updates_total"
K8gbInfobloxZoneUpdateErrorsTotal = "k8gb_infoblox_zone_update_errors_total"
K8gbEndpointStatusNum = "k8gb_endpoint_status_num"
Expand All @@ -64,6 +66,7 @@ type collectors struct {
K8gbGslbStatusCountForGeoip *prometheus.GaugeVec
K8gbGslbErrorsTotal *prometheus.CounterVec
K8gbGslbReconciliationLoopsTotal *prometheus.CounterVec
K8gbInfobloxRequestDuration *prometheus.HistogramVec
K8gbInfobloxZoneUpdatesTotal *prometheus.CounterVec
K8gbInfobloxZoneUpdateErrorsTotal *prometheus.CounterVec
K8gbInfobloxHeartbeatsTotal *prometheus.CounterVec
Expand All @@ -78,6 +81,21 @@ type PrometheusMetrics struct {
metrics collectors
}

// DNSProviderRequest is a label for histogram metric
type DNSProviderRequest string

const (
CreateZoneDelegated DNSProviderRequest = "ZoneCreate"
GetZoneDelegated DNSProviderRequest = "ZoneRead"
UpdateZoneDelegated DNSProviderRequest = "ZoneUpdate"
DeleteZoneDelegated DNSProviderRequest = "ZoneDelete"

CreateTXTRecord = "TXTRecordCreate"
GetTXTRecord = "TXTRecordRead"
UpdateTXTRecord = "TXTRecordUpdate"
DeleteTXTRecord = "TXTRecordDelete"
)

var regex = regexp.MustCompile("[A-Z]")

// newPrometheusMetrics creates new prometheus metrics instance
Expand Down Expand Up @@ -163,6 +181,11 @@ func (m *PrometheusMetrics) InfobloxIncrementHeartbeatError(gslb *k8gbv1beta1.Gs
m.metrics.K8gbInfobloxHeartbeatErrorsTotal.With(prometheus.Labels{"namespace": gslb.Namespace, "name": gslb.Name}).Inc()
}

func (m *PrometheusMetrics) InfobloxObserveRequestDuration(start time.Time, request DNSProviderRequest, success bool) {
duration := time.Since(start).Seconds()
m.metrics.K8gbInfobloxRequestDuration.With(prometheus.Labels{"request": string(request), "success": fmt.Sprintf("%t", success)}).Observe(duration)
}

func (m *PrometheusMetrics) SetRuntimeInfo(version, commit string) {
firstN := func(value string, n int) string {
if len(value) < n {
Expand Down Expand Up @@ -271,6 +294,15 @@ func (m *PrometheusMetrics) init() {
},
[]string{"namespace", "name", "status"},
)
m.metrics.K8gbInfobloxRequestDuration = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: K8gbInfobloxRequestDuration,
Help: "How long it took for Infoblox requests to complete, partitioned by request type. Round-trip time of http communication is included.",
Buckets: prometheus.ExponentialBuckets(.2, 4, 5),
},
[]string{"request", "success"},
)

m.metrics.K8gbInfobloxZoneUpdatesTotal = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: K8gbInfobloxZoneUpdatesTotal,
Expand Down
2 changes: 1 addition & 1 deletion controllers/providers/metrics/prometheus_result.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ type MetricResult struct {
value prometheus.Collector
}

// Get gets actual copy of metric defined by it's name
// Get gets actual copy of metric defined by its name
func (m *PrometheusMetrics) Get(name string) (r *MetricResult) {
return &MetricResult{value: m.registry()[name]}
}
Expand Down
4 changes: 2 additions & 2 deletions controllers/providers/metrics/prometheus_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,8 @@ func TestPrometheusRegistry(t *testing.T) {
items := []string{K8gbGslbErrorsTotal, K8gbGslbHealthyRecords, K8gbGslbReconciliationLoopsTotal,
K8gbGslbServiceStatusNum, K8gbGslbStatusCountForFailover, K8gbGslbStatusCountForRoundrobin,
K8gbGslbStatusCountForGeoIP, K8gbInfobloxHeartbeatsTotal, K8gbInfobloxHeartbeatErrorsTotal,
K8gbInfobloxZoneUpdatesTotal, K8gbInfobloxZoneUpdateErrorsTotal, K8gbEndpointStatusNum,
K8gbRuntimeInfo}
K8gbInfobloxRequestDuration, K8gbInfobloxZoneUpdatesTotal, K8gbInfobloxZoneUpdateErrorsTotal,
K8gbEndpointStatusNum, K8gbRuntimeInfo}
// act
registry := m.registry()
// assert
Expand Down