From 44549cd30c2af2ca70405f9d00fe8fb7f5a901e3 Mon Sep 17 00:00:00 2001 From: Hardikl <83282894+Hardikl@users.noreply.github.com> Date: Mon, 14 Nov 2022 21:31:24 +0530 Subject: [PATCH] fix: handle batching in shelf plugin (#1429) * fix: handle batching in shelf plugin * fix: handle batching at common place * fix: removed unused var * fix: few zapi calls dont support max-records param * fix: handled shelf changes in closure way * feat: handle ZAPI batching with closure (#1444) * feat: handle ZAPI batching with closure * feat: handle ZAPI batching with closure * feat: handle ZAPI batching with closure feat: Add LogSet support to plugins * fix: 7mode record check * fix: review comment * fix: log change Co-authored-by: Chris Grindstaff --- cmd/collectors/commonutils.go | 44 +---------- .../zapi/plugins/certificate/certificate.go | 11 ++- cmd/collectors/zapi/plugins/qtree/qtree.go | 3 +- .../zapi/plugins/security/security.go | 8 +- cmd/collectors/zapi/plugins/shelf/shelf.go | 78 +++++++++++-------- .../zapi/plugins/snapmirror/snapmirror.go | 6 +- cmd/collectors/zapi/plugins/svm/svm.go | 17 ++-- cmd/collectors/zapi/plugins/volume/volume.go | 20 ++--- cmd/poller/collector/collector.go | 2 +- conf/zapi/cdot/9.8.0/volume.yaml | 1 - pkg/api/ontapi/zapi/client.go | 71 +++++++++++++++++ pkg/conf/conf.go | 4 + 12 files changed, 156 insertions(+), 109 deletions(-) diff --git a/cmd/collectors/commonutils.go b/cmd/collectors/commonutils.go index 788b603b2..3a921e756 100644 --- a/cmd/collectors/commonutils.go +++ b/cmd/collectors/commonutils.go @@ -2,7 +2,6 @@ package collectors import ( "github.com/netapp/harvest/v2/cmd/tools/rest" - "github.com/netapp/harvest/v2/pkg/api/ontapi/zapi" "github.com/netapp/harvest/v2/pkg/logging" "github.com/netapp/harvest/v2/pkg/matrix" "github.com/netapp/harvest/v2/pkg/tree/node" @@ -11,6 +10,8 @@ import ( "time" ) +const DefaultBatchSize = "500" + func InvokeRestCall(client *rest.Client, href string, logger *logging.Logger) ([]gjson.Result, error) { result, err := rest.Fetch(client, href) if err != nil { @@ -25,45 +26,6 @@ func InvokeRestCall(client *rest.Client, href string, logger *logging.Logger) ([ return result, nil } -func InvokeZapiCall(client *zapi.Client, request *node.Node, logger *logging.Logger, tag string) ([]*node.Node, string, error) { - - var ( - result *node.Node - response []*node.Node - newTag string - err error - ) - - if tag != "" { - if result, newTag, err = client.InvokeBatchRequest(request, tag); err != nil { - return nil, "", err - } - } else { - if result, err = client.InvokeRequest(request); err != nil { - return nil, "", err - } - } - - if result == nil { - return nil, "", nil - } - - if x := result.GetChildS("attributes-list"); x != nil { - response = x.GetChildren() - } else if y := result.GetChildS("attributes"); y != nil { - // Check for non-list response - response = y.GetChildren() - } - - if len(response) == 0 { - return nil, "", nil - } - - logger.Trace().Int("object", len(response)).Msg("fetching") - - return response, newTag, nil -} - func UpdateProtectedFields(instance *matrix.Instance) { // check for group_type @@ -161,7 +123,7 @@ func GetDataInterval(param *node.Node, defaultInterval time.Duration) float64 { return defaultInterval.Seconds() } -// timestamp in micro seconds +// IsTimestampOlderThanDuration - timestamp units are micro seconds func IsTimestampOlderThanDuration(timestamp float64, duration time.Duration) bool { return time.Since(time.UnixMicro(int64(timestamp))) > duration } diff --git a/cmd/collectors/zapi/plugins/certificate/certificate.go b/cmd/collectors/zapi/plugins/certificate/certificate.go index 5c1834f91..434892d4a 100644 --- a/cmd/collectors/zapi/plugins/certificate/certificate.go +++ b/cmd/collectors/zapi/plugins/certificate/certificate.go @@ -58,12 +58,11 @@ func (my *Certificate) Init() error { return err } + my.batchSize = BatchSize if b := my.Params.GetChildContentS("batch_size"); b != "" { if _, err := strconv.Atoi(b); err == nil { my.batchSize = b } - } else { - my.batchSize = BatchSize } return nil @@ -204,7 +203,8 @@ func (my *Certificate) GetAdminVserver() (string, error) { vserverInfo := query.NewChildS("vserver-info", "") vserverInfo.NewChildS("vserver-type", "admin") - if result, _, err = collectors.InvokeZapiCall(my.client, request, my.Logger, ""); err != nil { + // Fetching only admin SVMs + if result, err = my.client.InvokeZapiCall(request); err != nil { return "", err } @@ -214,6 +214,7 @@ func (my *Certificate) GetAdminVserver() (string, error) { // This should be one iteration only as cluster can have one admin vserver for _, svm := range result { adminVserver = svm.GetChildContentS("vserver-name") + break } return adminVserver, nil } @@ -233,7 +234,8 @@ func (my *Certificate) GetSecuritySsl(adminSvm string) (string, error) { vserverInfo := query.NewChildS("vserver-ssl-info", "") vserverInfo.NewChildS("vserver", adminSvm) - if result, _, err = collectors.InvokeZapiCall(my.client, request, my.Logger, ""); err != nil { + // fetching data of only admin vservers + if result, err = my.client.InvokeZapiCall(request); err != nil { return "", err } @@ -243,6 +245,7 @@ func (my *Certificate) GetSecuritySsl(adminSvm string) (string, error) { // This should be one iteration only as cluster can have one admin vserver for _, ssl := range result { certificateSerial = ssl.GetChildContentS("certificate-serial-number") + break } return certificateSerial, nil } diff --git a/cmd/collectors/zapi/plugins/qtree/qtree.go b/cmd/collectors/zapi/plugins/qtree/qtree.go index fc33e7567..0dc1e4f04 100644 --- a/cmd/collectors/zapi/plugins/qtree/qtree.go +++ b/cmd/collectors/zapi/plugins/qtree/qtree.go @@ -92,13 +92,12 @@ func (my *Qtree) Init() error { my.Logger.Debug().Msgf("added data with %d metrics", len(my.data.GetMetrics())) // setup batchSize for request + my.batchSize = BatchSize if my.client.IsClustered() { if b := my.Params.GetChildContentS("batch_size"); b != "" { if _, err := strconv.Atoi(b); err == nil { my.batchSize = b } - } else { - my.batchSize = BatchSize } } diff --git a/cmd/collectors/zapi/plugins/security/security.go b/cmd/collectors/zapi/plugins/security/security.go index 1ead12610..55291dd23 100644 --- a/cmd/collectors/zapi/plugins/security/security.go +++ b/cmd/collectors/zapi/plugins/security/security.go @@ -106,7 +106,8 @@ func (my *Security) getSecurityConfig() (string, error) { request = node.NewXMLS("security-config-get") request.NewChildS("interface", "ssl") - if result, _, err = collectors.InvokeZapiCall(my.client, request, my.Logger, ""); err != nil { + // fetching only ssl interface + if result, err = my.client.InvokeZapiCall(request); err != nil { return "", err } @@ -116,6 +117,7 @@ func (my *Security) getSecurityConfig() (string, error) { for _, securityConfig := range result { fipsEnabled = securityConfig.GetChildContentS("is-fips-enabled") + break } return fipsEnabled, nil } @@ -153,7 +155,8 @@ func (my *Security) getEnabledValue(request *node.Node) (string, error) { err error ) - if result, _, err = collectors.InvokeZapiCall(my.client, request, my.Logger, ""); err != nil { + // fetching only telnet/rsh protocols + if result, err = my.client.InvokeZapiCall(request); err != nil { return "", err } @@ -163,6 +166,7 @@ func (my *Security) getEnabledValue(request *node.Node) (string, error) { for _, securityConfig := range result { enabled = securityConfig.GetChildContentS("enabled") + break } return enabled, nil diff --git a/cmd/collectors/zapi/plugins/shelf/shelf.go b/cmd/collectors/zapi/plugins/shelf/shelf.go index 903802896..bae39d92a 100644 --- a/cmd/collectors/zapi/plugins/shelf/shelf.go +++ b/cmd/collectors/zapi/plugins/shelf/shelf.go @@ -10,14 +10,18 @@ import ( "github.com/netapp/harvest/v2/pkg/matrix" "github.com/netapp/harvest/v2/pkg/tree/node" "github.com/netapp/harvest/v2/pkg/util" + "strconv" "strings" ) +const BatchSize = "500" + type Shelf struct { *plugin.AbstractPlugin data map[string]*matrix.Matrix instanceKeys map[string]string instanceLabels map[string]*dict.Dict + batchSize string client *zapi.Client query string } @@ -139,14 +143,24 @@ func (my *Shelf) Init() error { } my.Logger.Debug().Msgf("initialized with data [%d] objects", len(my.data)) + + // setup batchSize for request + my.batchSize = BatchSize + if my.client.IsClustered() { + if b := my.Params.GetChildContentS("batch_size"); b != "" { + if _, err := strconv.Atoi(b); err == nil { + my.batchSize = b + } + } + } return nil } func (my *Shelf) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) { var ( - result *node.Node err error + output []*matrix.Matrix ) if !my.client.IsClustered() { @@ -155,16 +169,18 @@ func (my *Shelf) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) { } } - if result, err = my.client.InvokeRequestString(my.query); err != nil { - return nil, err - } - // Set all global labels from zapi.go if already not exist for a := range my.instanceLabels { my.data[a].SetGlobalLabels(data.GetGlobalLabels()) } - var output []*matrix.Matrix + request := node.NewXMLS(my.query) + request.NewChildS("max-records", my.batchSize) + + result, err := my.client.InvokeZapiCall(request) + if err != nil { + return nil, err + } if my.client.IsClustered() { output, err = my.handleCMode(result) @@ -176,14 +192,15 @@ func (my *Shelf) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) { } if my.client.IsClustered() { - return my.calculateEnvironmentMetrics(output, data) - } else { - return output, nil + err := my.calculateEnvironmentMetrics(data) + if err != nil { + return nil, err + } } - + return output, nil } -func (my *Shelf) calculateEnvironmentMetrics(output []*matrix.Matrix, data *matrix.Matrix) ([]*matrix.Matrix, error) { +func (my *Shelf) calculateEnvironmentMetrics(data *matrix.Matrix) error { var err error shelfEnvironmentMetricMap := make(map[string]*shelfEnvironmentMetric, 0) for _, o := range my.data { @@ -326,25 +343,16 @@ func (my *Shelf) calculateEnvironmentMetrics(output []*matrix.Matrix, data *matr } } } - return output, nil + return nil } -func (my *Shelf) handleCMode(result *node.Node) ([]*matrix.Matrix, error) { +func (my *Shelf) handleCMode(shelves []*node.Node) ([]*matrix.Matrix, error) { var ( - shelves []*node.Node + output []*matrix.Matrix ) - if x := result.GetChildS("attributes-list"); x != nil { - shelves = x.GetChildren() - } - if len(shelves) == 0 { - return nil, errs.New(errs.ErrNoInstance, "no shelf instances found") - } - my.Logger.Debug().Msgf("fetching %d shelf counters", len(shelves)) - var output []*matrix.Matrix - // Purge and reset data for _, data1 := range my.data { data1.PurgeInstances() @@ -399,7 +407,7 @@ func (my *Shelf) handleCMode(result *node.Node) ([]*matrix.Matrix, error) { instance.SetLabel("shelf", shelfName) instance.SetLabel("shelf_id", shelfID) - // Each child would have different possible values which is ugly way to write all of them, + // Each child would have different possible values which is an ugly way to write all of them, // so normal value would be mapped to 1 and rest all are mapped to 0. if instance.GetLabel("status") == "normal" { _ = statusMetric.SetValueInt64(instance, 1) @@ -431,19 +439,25 @@ func (my *Shelf) handleCMode(result *node.Node) ([]*matrix.Matrix, error) { return output, nil } -func (my *Shelf) handle7Mode(result *node.Node) ([]*matrix.Matrix, error) { +func (my *Shelf) handle7Mode(result []*node.Node) ([]*matrix.Matrix, error) { var ( shelves []*node.Node channels []*node.Node + output []*matrix.Matrix ) - //fallback to 7mode - channels = result.SearchChildren([]string{"shelf-environ-channel-info"}) - if len(channels) == 0 { - return nil, errs.New(errs.ErrNoInstance, "no channels found") + // Result would be the zapi response itself with only one record. + if len(result) != 1 { + my.Logger.Debug().Msg("no shelves found") + return output, nil } + // fallback to 7mode + channels = result[0].SearchChildren([]string{"shelf-environ-channel-info"}) - var output []*matrix.Matrix + if len(channels) == 0 { + my.Logger.Debug().Msg("no channels found") + return output, nil + } // Purge and reset data for _, data1 := range my.data { @@ -456,7 +470,7 @@ func (my *Shelf) handle7Mode(result *node.Node) ([]*matrix.Matrix, error) { shelves = channel.SearchChildren([]string{"shelf-environ-shelf-list", "shelf-environ-shelf-info"}) if len(shelves) == 0 { - my.Logger.Warn().Str("channel", channelName).Msg("no shelves found") + my.Logger.Debug().Str("channel", channelName).Msg("no shelves found") continue } @@ -504,7 +518,7 @@ func (my *Shelf) handle7Mode(result *node.Node) ([]*matrix.Matrix, error) { instance.SetLabel("shelf_id", shelfID) instance.SetLabel("channel", channelName) - // Each child would have different possible values which is ugly way to write all of them, + // Each child would have different possible values which is an ugly way to write all of them, // so normal value would be mapped to 1 and rest all are mapped to 0. if instance.GetLabel("status") == "normal" { _ = statusMetric.SetValueInt64(instance, 1) diff --git a/cmd/collectors/zapi/plugins/snapmirror/snapmirror.go b/cmd/collectors/zapi/plugins/snapmirror/snapmirror.go index bbd7cda97..5631d069f 100644 --- a/cmd/collectors/zapi/plugins/snapmirror/snapmirror.go +++ b/cmd/collectors/zapi/plugins/snapmirror/snapmirror.go @@ -81,7 +81,7 @@ func (my *SnapMirror) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) { } for _, instance := range data.GetInstances() { - // Zapi call with `expand=true` would gives all the constituent's relationships as well, which we don't want to export. + // Zapi call with `expand=true` returns all the constituent's relationships. We do not want to export them. if match := flexgroupConstituentName.FindStringSubmatch(instance.GetLabel("destination_volume")); len(match) == 3 { instance.SetExportable(false) continue @@ -185,6 +185,7 @@ func (my *SnapMirror) getSVMPeerData(cluster string) error { ) request := node.NewXMLS("vserver-peer-get-iter") + request.NewChildS("max-records", collectors.DefaultBatchSize) // Fetching only remote vserver-peer query := request.NewChildS("query", "") vserverPeerInfo := query.NewChildS("vserver-peer-info", "") @@ -193,7 +194,8 @@ func (my *SnapMirror) getSVMPeerData(cluster string) error { // Clean svmPeerMap map my.svmPeerDataMap = make(map[string]Peer) - if result, _, err = collectors.InvokeZapiCall(my.client, request, my.Logger, ""); err != nil { + // fetching only remote vserver peer data + if result, err = my.client.InvokeZapiCall(request); err != nil { return err } diff --git a/cmd/collectors/zapi/plugins/svm/svm.go b/cmd/collectors/zapi/plugins/svm/svm.go index cd1a61ac6..3217cea5c 100644 --- a/cmd/collectors/zapi/plugins/svm/svm.go +++ b/cmd/collectors/zapi/plugins/svm/svm.go @@ -77,13 +77,13 @@ func (my *SVM) Init() error { return err } + my.batchSize = BatchSize if b := my.Params.GetChildContentS("batch_size"); b != "" { if _, err := strconv.Atoi(b); err == nil { my.batchSize = b my.Logger.Info().Str("BatchSize", my.batchSize).Msg("using batch-size") } } else { - my.batchSize = BatchSize my.Logger.Trace().Str("BatchSize", BatchSize).Msg("Using default batch-size") } @@ -221,7 +221,7 @@ func (my *SVM) GetAuditProtocols() (map[string]string, error) { request = node.NewXMLS("fileservice-audit-config-get-iter") request.NewChildS("max-records", my.batchSize) - if result, _, err = collectors.InvokeZapiCall(my.client, request, my.Logger, ""); err != nil { + if result, err = my.client.InvokeZapiCall(request); err != nil { return nil, err } @@ -250,7 +250,7 @@ func (my *SVM) GetCifsProtocols() (map[string]string, error) { request = node.NewXMLS("cifs-security-get-iter") request.NewChildS("max-records", my.batchSize) - if result, _, err = collectors.InvokeZapiCall(my.client, request, my.Logger, ""); err != nil { + if result, err = my.client.InvokeZapiCall(request); err != nil { return nil, err } @@ -281,7 +281,7 @@ func (my *SVM) GetNSSwitchInfo() (map[string]nsswitch, error) { request = node.NewXMLS("nameservice-nsswitch-get-iter") request.NewChildS("max-records", my.batchSize) - if result, _, err = collectors.InvokeZapiCall(my.client, request, my.Logger, ""); err != nil { + if result, err = my.client.InvokeZapiCall(request); err != nil { return nil, err } @@ -319,7 +319,7 @@ func (my *SVM) GetNisInfo() (map[string]string, error) { request = node.NewXMLS("nis-get-iter") request.NewChildS("max-records", my.batchSize) - if result, _, err = collectors.InvokeZapiCall(my.client, request, my.Logger, ""); err != nil { + if result, err = my.client.InvokeZapiCall(request); err != nil { return nil, err } @@ -347,8 +347,7 @@ func (my *SVM) GetCifsEnabled() (map[string]bool, error) { request = node.NewXMLS("cifs-server-get-iter") request.NewChildS("max-records", my.batchSize) - - if result, _, err = collectors.InvokeZapiCall(my.client, request, my.Logger, ""); err != nil { + if result, err = my.client.InvokeZapiCall(request); err != nil { return nil, err } @@ -377,7 +376,7 @@ func (my *SVM) GetNfsEnabled() (map[string]string, error) { request = node.NewXMLS("nfs-service-get-iter") request.NewChildS("max-records", my.batchSize) - if result, _, err = collectors.InvokeZapiCall(my.client, request, my.Logger, ""); err != nil { + if result, err = my.client.InvokeZapiCall(request); err != nil { return nil, err } @@ -406,7 +405,7 @@ func (my *SVM) GetSSHData() (map[string]string, error) { request = node.NewXMLS("security-ssh-get-iter") request.NewChildS("max-records", my.batchSize) - if result, _, err = collectors.InvokeZapiCall(my.client, request, my.Logger, ""); err != nil { + if result, err = my.client.InvokeZapiCall(request); err != nil { return nil, err } diff --git a/cmd/collectors/zapi/plugins/volume/volume.go b/cmd/collectors/zapi/plugins/volume/volume.go index f785207f2..a234b24ae 100644 --- a/cmd/collectors/zapi/plugins/volume/volume.go +++ b/cmd/collectors/zapi/plugins/volume/volume.go @@ -13,13 +13,11 @@ import ( "time" ) -const BatchSize = "500" const DefaultPluginDuration = 30 * time.Minute const DefaultDataPollDuration = 3 * time.Minute type Volume struct { *plugin.AbstractPlugin - batchSize string pluginInvocationRate int currentVal int client *zapi.Client @@ -59,17 +57,6 @@ func (my *Volume) Init() error { my.Logger.Error().Err(err).Stack().Msg("Failed while setting the plugin interval") } - // batching the request - if b := my.Params.GetChildContentS("batch_size"); b != "" { - if _, err := strconv.Atoi(b); err == nil { - my.batchSize = b - my.Logger.Info().Str("BatchSize", my.batchSize).Msg("using batch-size") - } - } else { - my.batchSize = BatchSize - my.Logger.Trace().Str("BatchSize", BatchSize).Msg("Using default batch-size") - } - return nil } @@ -124,13 +111,15 @@ func (my *Volume) getEncryptedDisks() ([]string, error) { ) request := node.NewXMLS("disk-encrypt-get-iter") + request.NewChildS("max-records", collectors.DefaultBatchSize) //algorithm is -- Protection mode needs to be DATA or FULL // Fetching rest of them and add as query := request.NewChildS("query", "") encryptInfoQuery := query.NewChildS("disk-encrypt-info", "") encryptInfoQuery.NewChildS("protection-mode", "open|part|miss") - if result, _, err = collectors.InvokeZapiCall(my.client, request, my.Logger, ""); err != nil { + // fetching only disks whose protection-mode is open/part/miss + if result, err = my.client.InvokeZapiCall(request); err != nil { return nil, err } @@ -166,9 +155,10 @@ func (my *Volume) getAggrDiskMapping() (map[string]aggrData, error) { ) request := node.NewXMLS("aggr-status-get-iter") + request.NewChildS("max-records", collectors.DefaultBatchSize) aggrsDisksMap = make(map[string]aggrData) - if result, _, err = collectors.InvokeZapiCall(my.client, request, my.Logger, ""); err != nil { + if result, err = my.client.InvokeZapiCall(request); err != nil { return nil, err } diff --git a/cmd/poller/collector/collector.go b/cmd/poller/collector/collector.go index 4e8dc8e42..2a07b3266 100644 --- a/cmd/poller/collector/collector.go +++ b/cmd/poller/collector/collector.go @@ -394,7 +394,7 @@ func (c *AbstractCollector) Start(wg *sync.WaitGroup) { for k, v := range c.Plugins { for _, plg := range v { if pluginData, err := plg.Run(data[k]); err != nil { - c.Logger.Error().Stack().Err(err).Msgf("plugin [%s]: ", plg.GetName()) + c.Logger.Error().Err(err).Str("plugin", plg.GetName()).Send() } else if pluginData != nil { results = append(results, pluginData...) c.Logger.Debug().Msgf("plugin [%s] added (%d) data", plg.GetName(), len(pluginData)) diff --git a/conf/zapi/cdot/9.8.0/volume.yaml b/conf/zapi/cdot/9.8.0/volume.yaml index 01c0e9182..e337b8d0b 100644 --- a/conf/zapi/cdot/9.8.0/volume.yaml +++ b/conf/zapi/cdot/9.8.0/volume.yaml @@ -70,7 +70,6 @@ plugins: Volume: schedule: - data: 900s # should be multiple of data poll duration - #batch_size: "50" LabelAgent: # metric label zapi_value rest_value `default_value` value_to_num: diff --git a/pkg/api/ontapi/zapi/client.go b/pkg/api/ontapi/zapi/client.go index 98b231539..e7f1979df 100644 --- a/pkg/api/ontapi/zapi/client.go +++ b/pkg/api/ontapi/zapi/client.go @@ -74,6 +74,16 @@ func New(poller conf.Poller) (*Client, error) { } else { url = "https://" + addr + ":443/servlets/netapp.servlets.admin.XMLrequest_filer" } + + if poller.LogSet != nil { + for _, name := range *poller.LogSet { + if name == "Zapi" || name == "ZapiPerf" { + client.logZapi = true + break + } + } + } + // create a request object that will be used for later requests if request, err = http.NewRequest("POST", url, nil); err != nil { return nil, err @@ -280,6 +290,7 @@ func (c *Client) buildRequest(query *node.Node, forceCluster bool) error { ) request = node.NewXMLS("netapp") + //goland:noinspection HttpUrlsUsage request.NewAttrS("xmlns", "http://www.netapp.com/filer/admin") request.NewAttrS("version", c.apiVersion) // optionally use fviler-tunneling, this option is never used in Harvest @@ -299,6 +310,66 @@ func (c *Client) buildRequest(query *node.Node, forceCluster bool) error { return nil } +// InvokeZapi will issue API requests with batching +// The method bails on the first error +func (c *Client) InvokeZapi(request *node.Node, handle func([]*node.Node) error) error { + var output []*node.Node + tag := "initial" + + for { + var ( + result *node.Node + response []*node.Node + err error + ) + + if result, tag, err = c.InvokeBatchRequest(request, tag); err != nil { + return err + } + + if result == nil { + break + } + + // for 7mode, the output will be the zapi response since 7mode does not support pagination + if !c.IsClustered() { + response = append(response, result) + // 7mode does not support pagination. set the tag an empty string to break the for loop + tag = "" + } else if x := result.GetChildS("attributes-list"); x != nil { + response = x.GetChildren() + } else if y := result.GetChildS("attributes"); y != nil { + // Check for non-list response + response = y.GetChildren() + } + + if len(response) == 0 { + break + } + err = handle(response) + if err != nil { + return err + } + } + + c.Logger.Trace().Int("object", len(output)).Msg("fetching") + + return nil +} + +// InvokeZapiCall will issue API requests with batching +func (c *Client) InvokeZapiCall(request *node.Node) ([]*node.Node, error) { + var output []*node.Node + err := c.InvokeZapi(request, func(response []*node.Node) error { + output = append(output, response...) + return nil + }) + if err != nil { + return nil, err + } + return output, nil +} + // Invoke will issue the API request and return server response // this method should only be called after building the request func (c *Client) Invoke() (*node.Node, error) { diff --git a/pkg/conf/conf.go b/pkg/conf/conf.go index 057323fb2..a6a2e2cf7 100644 --- a/pkg/conf/conf.go +++ b/pkg/conf/conf.go @@ -444,6 +444,10 @@ func ZapiPoller(n *node.Node) Poller { if tlsMinVersion := n.GetChildContentS("tls_min_version"); tlsMinVersion != "" { p.TLSMinVersion = tlsMinVersion } + if logSet := n.GetChildS("log"); logSet != nil { + names := logSet.GetAllChildNamesS() + p.LogSet = &names + } return p }