From 07d480c146af6ff530e9c1ec3d580630e90dc415 Mon Sep 17 00:00:00 2001 From: Jirka Kremser Date: Wed, 15 Dec 2021 14:53:26 +0000 Subject: [PATCH] Add histogram metrics for infoblox calls Signed-off-by: Jirka Kremser --- controllers/providers/dns/infoblox.go | 94 +++++++++++++++---- controllers/providers/metrics/prometheus.go | 32 +++++++ .../providers/metrics/prometheus_result.go | 2 +- .../providers/metrics/prometheus_test.go | 4 +- 4 files changed, 110 insertions(+), 22 deletions(-) diff --git a/controllers/providers/dns/infoblox.go b/controllers/providers/dns/infoblox.go index e52a87a90f..99e82462db 100644 --- a/controllers/providers/dns/infoblox.go +++ b/controllers/providers/dns/infoblox.go @@ -24,7 +24,7 @@ import ( externaldns "sigs.k8s.io/external-dns/endpoint" - ibclient "github.com/infobloxopen/infoblox-go-client" + ibcl "github.com/infobloxopen/infoblox-go-client" k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1" "github.com/k8gb-io/k8gb/controllers/depresolver" "github.com/k8gb-io/k8gb/controllers/providers/assistant" @@ -47,7 +47,7 @@ func NewInfobloxDNS(config depresolver.Config, assistant assistant.Assistant, cl } } -func (p *InfobloxProvider) sanitizeDelegateZone(local, upstream []ibclient.NameServer) []ibclient.NameServer { +func (p *InfobloxProvider) sanitizeDelegateZone(local, upstream []ibcl.NameServer) []ibcl.NameServer { // Drop own records for straight away update // And ensure local entries are up to date // And final list is sorted @@ -70,14 +70,14 @@ func (p *InfobloxProvider) CreateZoneDelegationForExternalDNS(gslb *k8gbv1beta1. m.InfobloxIncrementZoneUpdateError(gslb) return err } - var delegateTo []ibclient.NameServer + var delegateTo []ibcl.NameServer for _, address := range addresses { - nameServer := ibclient.NameServer{Address: address, Name: p.config.GetClusterNSName()} + nameServer := ibcl.NameServer{Address: address, Name: p.config.GetClusterNSName()} delegateTo = append(delegateTo, nameServer) } - findZone, err := objMgr.GetZoneDelegated(p.config.DNSZone) + findZone, err := p.getZoneDelegated(objMgr, p.config.DNSZone) if err != nil { m.InfobloxIncrementZoneUpdateError(gslb) return err @@ -124,7 +124,7 @@ func (p *InfobloxProvider) CreateZoneDelegationForExternalDNS(gslb *k8gbv1beta1. Str("DNS zone", p.config.DNSZone). Str("server list", fmt.Sprintf("%v", currentList)). Msg("Updating delegated zone with the server list") - _, err = objMgr.UpdateZoneDelegated(findZone.Ref, currentList) + _, err = p.updateZoneDelegated(objMgr, findZone.Ref, currentList) if err != nil { m.InfobloxIncrementZoneUpdateError(gslb) return err @@ -140,7 +140,7 @@ func (p *InfobloxProvider) CreateZoneDelegationForExternalDNS(gslb *k8gbv1beta1. log.Debug(). Str("records", fmt.Sprintf("%v", delegateTo)). Msg("Delegated records") - _, err = objMgr.CreateZoneDelegated(p.config.DNSZone, delegateTo) + _, err = p.createZoneDelegated(objMgr, p.config.DNSZone, delegateTo) if err != nil { m.InfobloxIncrementZoneUpdateError(gslb) return err @@ -158,7 +158,7 @@ func (p *InfobloxProvider) Finalize(gslb *k8gbv1beta1.Gslb) error { if err != nil { return err } - findZone, err := objMgr.GetZoneDelegated(p.config.DNSZone) + findZone, err := p.getZoneDelegated(objMgr, p.config.DNSZone) if err != nil { return err } @@ -172,7 +172,7 @@ func (p *InfobloxProvider) Finalize(gslb *k8gbv1beta1.Gslb) error { log.Info(). Str("DNS zone", p.config.DNSZone). Msg("Deleting delegated zone") - _, err := objMgr.DeleteZoneDelegated(findZone.Ref) + _, err := p.deleteZoneDelegated(objMgr, findZone.Ref) if err != nil { return err } @@ -180,7 +180,7 @@ func (p *InfobloxProvider) Finalize(gslb *k8gbv1beta1.Gslb) error { } heartbeatTXTName := p.config.GetClusterHeartbeatFQDN(gslb.Name) - findTXT, err := objMgr.GetTXTRecord(heartbeatTXTName) + findTXT, err := p.getTXTRecord(objMgr, heartbeatTXTName) if err != nil { return err } @@ -190,7 +190,7 @@ func (p *InfobloxProvider) Finalize(gslb *k8gbv1beta1.Gslb) error { log.Info(). Str("TXT records", heartbeatTXTName). Msg("Deleting split brain TXT record") - _, err := objMgr.DeleteTXTRecord(findTXT.Ref) + _, err := p.deleteTXTRecord(objMgr, findTXT.Ref) if err != nil { return err } @@ -215,11 +215,11 @@ func (p *InfobloxProvider) String() string { return "Infoblox" } -func (p *InfobloxProvider) saveHeartbeatTXTRecord(objMgr *ibclient.ObjectManager, gslb *k8gbv1beta1.Gslb) (err error) { - var heartbeatTXTRecord *ibclient.RecordTXT +func (p *InfobloxProvider) saveHeartbeatTXTRecord(objMgr *ibcl.ObjectManager, gslb *k8gbv1beta1.Gslb) (err error) { + var heartbeatTXTRecord *ibcl.RecordTXT edgeTimestamp := fmt.Sprint(time.Now().UTC().Format("2006-01-02T15:04:05")) heartbeatTXTName := p.config.GetClusterHeartbeatFQDN(gslb.Name) - heartbeatTXTRecord, err = objMgr.GetTXTRecord(heartbeatTXTName) + heartbeatTXTRecord, err = p.getTXTRecord(objMgr, heartbeatTXTName) if err != nil { return } @@ -227,7 +227,7 @@ func (p *InfobloxProvider) saveHeartbeatTXTRecord(objMgr *ibclient.ObjectManager log.Info(). Str("HeartbeatTXTName", heartbeatTXTName). Msg("Creating split brain TXT record") - _, err = objMgr.CreateTXTRecord(heartbeatTXTName, edgeTimestamp, uint(gslb.Spec.Strategy.DNSTtlSeconds), "default") + _, err = p.createTXTRecord(objMgr, heartbeatTXTName, edgeTimestamp, uint(gslb.Spec.Strategy.DNSTtlSeconds)) if err != nil { m.InfobloxIncrementHeartbeatError(gslb) return @@ -236,7 +236,7 @@ func (p *InfobloxProvider) saveHeartbeatTXTRecord(objMgr *ibclient.ObjectManager log.Info(). Str("HeartbeatTXTName", heartbeatTXTName). Msg("Updating split brain TXT record") - _, err = objMgr.UpdateTXTRecord(heartbeatTXTName, edgeTimestamp) + _, err = p.updateTXTRecord(objMgr, heartbeatTXTName, edgeTimestamp) if err != nil { m.InfobloxIncrementHeartbeatError(gslb) return @@ -246,7 +246,7 @@ func (p *InfobloxProvider) saveHeartbeatTXTRecord(objMgr *ibclient.ObjectManager return } -func (p *InfobloxProvider) checkZoneDelegated(findZone *ibclient.ZoneDelegated) error { +func (p *InfobloxProvider) checkZoneDelegated(findZone *ibcl.ZoneDelegated) error { if findZone.Fqdn != p.config.DNSZone { err := fmt.Errorf("delegated zone returned from infoblox(%s) does not match requested gslb zone(%s)", findZone.Fqdn, p.config.DNSZone) return err @@ -254,8 +254,8 @@ func (p *InfobloxProvider) checkZoneDelegated(findZone *ibclient.ZoneDelegated) return nil } -func (p *InfobloxProvider) filterOutDelegateTo(delegateTo []ibclient.NameServer, fqdn string) (result []ibclient.NameServer) { - result = make([]ibclient.NameServer, 0) +func (p *InfobloxProvider) filterOutDelegateTo(delegateTo []ibcl.NameServer, fqdn string) (result []ibcl.NameServer) { + result = make([]ibcl.NameServer, 0) for _, v := range delegateTo { if v.Name != fqdn { @@ -264,3 +264,59 @@ func (p *InfobloxProvider) filterOutDelegateTo(delegateTo []ibclient.NameServer, } return } + +func (p *InfobloxProvider) createZoneDelegated(o *ibcl.ObjectManager, fqdn string, d []ibcl.NameServer) (res *ibcl.ZoneDelegated, err error) { + start := time.Now() + res, err = o.CreateZoneDelegated(fqdn, d) + m.InfobloxObserveRequestDuration(start, metrics.CreateZoneDelegated, err == nil) + return +} + +func (p *InfobloxProvider) getZoneDelegated(o *ibcl.ObjectManager, fqdn string) (res *ibcl.ZoneDelegated, err error) { + start := time.Now() + res, err = o.GetZoneDelegated(fqdn) + m.InfobloxObserveRequestDuration(start, metrics.GetZoneDelegated, err == nil) + return +} + +func (p *InfobloxProvider) updateZoneDelegated(o *ibcl.ObjectManager, fqdn string, d []ibcl.NameServer) (res *ibcl.ZoneDelegated, err error) { + start := time.Now() + res, err = o.UpdateZoneDelegated(fqdn, d) + m.InfobloxObserveRequestDuration(start, metrics.UpdateZoneDelegated, err == nil) + return +} + +func (p *InfobloxProvider) deleteZoneDelegated(o *ibcl.ObjectManager, fqdn string) (res string, err error) { + start := time.Now() + res, err = o.DeleteZoneDelegated(fqdn) + m.InfobloxObserveRequestDuration(start, metrics.DeleteZoneDelegated, err == nil) + return +} + +func (p *InfobloxProvider) createTXTRecord(o *ibcl.ObjectManager, name string, text string, ttl uint) (res *ibcl.RecordTXT, err error) { + start := time.Now() + res, err = o.CreateTXTRecord(name, text, ttl, "default") + m.InfobloxObserveRequestDuration(start, metrics.CreateTXTRecord, err == nil) + return +} + +func (p *InfobloxProvider) getTXTRecord(o *ibcl.ObjectManager, name string) (res *ibcl.RecordTXT, err error) { + start := time.Now() + res, err = o.GetTXTRecord(name) + m.InfobloxObserveRequestDuration(start, metrics.GetTXTRecord, err == nil) + return +} + +func (p *InfobloxProvider) updateTXTRecord(o *ibcl.ObjectManager, name string, text string) (res *ibcl.RecordTXT, err error) { + start := time.Now() + res, err = o.UpdateTXTRecord(name, text) + m.InfobloxObserveRequestDuration(start, metrics.UpdateTXTRecord, err == nil) + return +} + +func (p *InfobloxProvider) deleteTXTRecord(o *ibcl.ObjectManager, name string) (res string, err error) { + start := time.Now() + res, err = o.DeleteTXTRecord(name) + m.InfobloxObserveRequestDuration(start, metrics.DeleteTXTRecord, err == nil) + return +} diff --git a/controllers/providers/metrics/prometheus.go b/controllers/providers/metrics/prometheus.go index 9668fd1ddd..db6ebd37a7 100644 --- a/controllers/providers/metrics/prometheus.go +++ b/controllers/providers/metrics/prometheus.go @@ -24,6 +24,7 @@ import ( "runtime" "strings" "sync" + "time" externaldns "sigs.k8s.io/external-dns/endpoint" @@ -49,6 +50,7 @@ const ( K8gbGslbStatusCountForGeoIP = "k8gb_gslb_status_count_for_geoip" K8gbInfobloxHeartbeatsTotal = "k8gb_infoblox_heartbeats_total" K8gbInfobloxHeartbeatErrorsTotal = "k8gb_infoblox_heartbeat_errors_total" + K8gbInfobloxRequestDuration = "k8gb_infoblox_request_duration" K8gbInfobloxZoneUpdatesTotal = "k8gb_infoblox_zone_updates_total" K8gbInfobloxZoneUpdateErrorsTotal = "k8gb_infoblox_zone_update_errors_total" K8gbEndpointStatusNum = "k8gb_endpoint_status_num" @@ -64,6 +66,7 @@ type collectors struct { K8gbGslbStatusCountForGeoip *prometheus.GaugeVec K8gbGslbErrorsTotal *prometheus.CounterVec K8gbGslbReconciliationLoopsTotal *prometheus.CounterVec + K8gbInfobloxRequestDuration *prometheus.HistogramVec K8gbInfobloxZoneUpdatesTotal *prometheus.CounterVec K8gbInfobloxZoneUpdateErrorsTotal *prometheus.CounterVec K8gbInfobloxHeartbeatsTotal *prometheus.CounterVec @@ -78,6 +81,21 @@ type PrometheusMetrics struct { metrics collectors } +// DNSProviderRequest is a label for histogram metric +type DNSProviderRequest string + +const ( + CreateZoneDelegated DNSProviderRequest = "ZoneCreate" + GetZoneDelegated DNSProviderRequest = "ZoneRead" + UpdateZoneDelegated DNSProviderRequest = "ZoneUpdate" + DeleteZoneDelegated DNSProviderRequest = "ZoneDelete" + + CreateTXTRecord = "TXTRecordCreate" + GetTXTRecord = "TXTRecordRead" + UpdateTXTRecord = "TXTRecordUpdate" + DeleteTXTRecord = "TXTRecordDelete" +) + var regex = regexp.MustCompile("[A-Z]") // newPrometheusMetrics creates new prometheus metrics instance @@ -163,6 +181,11 @@ func (m *PrometheusMetrics) InfobloxIncrementHeartbeatError(gslb *k8gbv1beta1.Gs m.metrics.K8gbInfobloxHeartbeatErrorsTotal.With(prometheus.Labels{"namespace": gslb.Namespace, "name": gslb.Name}).Inc() } +func (m *PrometheusMetrics) InfobloxObserveRequestDuration(start time.Time, request DNSProviderRequest, success bool) { + duration := time.Since(start).Seconds() + m.metrics.K8gbInfobloxRequestDuration.With(prometheus.Labels{"request": string(request), "success": fmt.Sprintf("%t", success)}).Observe(duration) +} + func (m *PrometheusMetrics) SetRuntimeInfo(version, commit string) { firstN := func(value string, n int) string { if len(value) < n { @@ -271,6 +294,15 @@ func (m *PrometheusMetrics) init() { }, []string{"namespace", "name", "status"}, ) + m.metrics.K8gbInfobloxRequestDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: K8gbInfobloxRequestDuration, + Help: "How long it took for Infoblox requests to complete, partitioned by request type. Round-trip time of http communication is included.", + Buckets: prometheus.ExponentialBuckets(.2, 4, 5), + }, + []string{"request", "success"}, + ) + m.metrics.K8gbInfobloxZoneUpdatesTotal = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: K8gbInfobloxZoneUpdatesTotal, diff --git a/controllers/providers/metrics/prometheus_result.go b/controllers/providers/metrics/prometheus_result.go index d63d782593..b31938acf7 100644 --- a/controllers/providers/metrics/prometheus_result.go +++ b/controllers/providers/metrics/prometheus_result.go @@ -26,7 +26,7 @@ type MetricResult struct { value prometheus.Collector } -// Get gets actual copy of metric defined by it's name +// Get gets actual copy of metric defined by its name func (m *PrometheusMetrics) Get(name string) (r *MetricResult) { return &MetricResult{value: m.registry()[name]} } diff --git a/controllers/providers/metrics/prometheus_test.go b/controllers/providers/metrics/prometheus_test.go index bd2c21904a..672be59060 100644 --- a/controllers/providers/metrics/prometheus_test.go +++ b/controllers/providers/metrics/prometheus_test.go @@ -77,8 +77,8 @@ func TestPrometheusRegistry(t *testing.T) { items := []string{K8gbGslbErrorsTotal, K8gbGslbHealthyRecords, K8gbGslbReconciliationLoopsTotal, K8gbGslbServiceStatusNum, K8gbGslbStatusCountForFailover, K8gbGslbStatusCountForRoundrobin, K8gbGslbStatusCountForGeoIP, K8gbInfobloxHeartbeatsTotal, K8gbInfobloxHeartbeatErrorsTotal, - K8gbInfobloxZoneUpdatesTotal, K8gbInfobloxZoneUpdateErrorsTotal, K8gbEndpointStatusNum, - K8gbRuntimeInfo} + K8gbInfobloxRequestDuration, K8gbInfobloxZoneUpdatesTotal, K8gbInfobloxZoneUpdateErrorsTotal, + K8gbEndpointStatusNum, K8gbRuntimeInfo} // act registry := m.registry() // assert