Skip to content

Commit

Permalink
feat: address review comments
Browse files Browse the repository at this point in the history
  • Loading branch information
@rahulguptajss
rahulguptajss committed Jan 12, 2023
1 parent eca86be commit 51b0d7c
Show file tree
Hide file tree
Showing 10 changed files with 13,868 additions and 13,231 deletions.
56 changes: 37 additions & 19 deletions cmd/tools/rest/counter.go → cmd/tools/generate/counter.go
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
package rest
package generate

import (
"fmt"
"github.com/netapp/harvest/v2/cmd/tools/rest"
"github.com/netapp/harvest/v2/pkg/api/ontapi/zapi"
"github.com/netapp/harvest/v2/pkg/tree"
"github.com/netapp/harvest/v2/pkg/tree/node"
Expand All @@ -15,6 +16,7 @@ import (
"sort"
"strings"
"text/template"
"time"
)

var (
Expand All @@ -32,8 +34,14 @@ type Counters struct {
C []Counter `yaml:"counters"`
}

type CounterMetaData struct {
Date string
OntapVersion string
}

type CounterTemplate struct {
Counters []Counter
Counters []Counter
CounterMetaData CounterMetaData
}

type Counter struct {
Expand All @@ -44,17 +52,19 @@ type Counter struct {
RestTemplate string `yaml:"RestTemplate"`
RestUnit string `yaml:"RestUnit"`
RestType string `yaml:"RestType"`
RestBaseCounter string `yaml:"RestBaseCounter"`
ZapiAPI string `yaml:"ZapiAPI"`
ZapiONTAPCounter string `yaml:"ZapiONTAPCounter"`
ZapiTemplate string `yaml:"ZapiTemplate"`
ZapiUnit string `yaml:"ZapiUnit"`
ZapiType string `yaml:"ZapiType"`
ZapiBaseCounter string `yaml:"ZapiBaseCounter"`
}

// readSwaggerJSON downloads poller swagger and convert to json format
func readSwaggerJSON() []byte {
var f []byte
path, err := readOrDownloadSwagger()
path, err := rest.ReadOrDownloadSwagger(opts.Poller)
if err != nil {
log.Fatal("failed to download swagger:", err)
return nil
Expand Down Expand Up @@ -88,12 +98,12 @@ func searchDescriptionSwagger(objName string, ontapCounterName string) string {
}

// processRestCounters parse rest and restperf templates
func processRestCounters(client *Client) map[string]Counter {
restPerfCounters := visitRestTemplates("conf/restperf", client, func(path string, client *Client) map[string]Counter {
func processRestCounters(client *rest.Client) map[string]Counter {
restPerfCounters := visitRestTemplates("conf/restperf", client, func(path string, client *rest.Client) map[string]Counter {
return processRestPerfCounters(path, client)
})

restCounters := visitRestTemplates("conf/rest", client, func(path string, client *Client) map[string]Counter {
restCounters := visitRestTemplates("conf/rest", client, func(path string, client *rest.Client) map[string]Counter {
return processRestConfigCounters(path)
})

Expand Down Expand Up @@ -211,11 +221,12 @@ func processRestConfigCounters(path string) map[string]Counter {
// processZAPIPerfCounters process ZapiPerf counters
func processZAPIPerfCounters(path string, client *zapi.Client) map[string]Counter {
var (
counters = make(map[string]Counter)
request, response *node.Node
zapiUnitMap = make(map[string]string)
zapiTypeMap = make(map[string]string)
zapiDescMap = make(map[string]string)
counters = make(map[string]Counter)
request, response *node.Node
zapiUnitMap = make(map[string]string)
zapiTypeMap = make(map[string]string)
zapiDescMap = make(map[string]string)
zapiBaseCounterMap = make(map[string]string)
)
t, err := tree.ImportYaml(path)
if t == nil || err != nil {
Expand Down Expand Up @@ -256,6 +267,7 @@ func processZAPIPerfCounters(path string, client *zapi.Client) map[string]Counte
zapiUnitMap[name] = counter.GetChildContentS("unit")
zapiDescMap[name] = updateDescription(counter.GetChildContentS("desc"))
zapiTypeMap[name] = ty
zapiBaseCounterMap[name] = counter.GetChildContentS("base-counter")
}
}
}
Expand All @@ -282,6 +294,7 @@ func processZAPIPerfCounters(path string, client *zapi.Client) map[string]Counte
ZapiAPI: strings.Join([]string{"perf-object-get-instances", query}, " "),
ZapiType: zapiTypeMap[name],
ZapiUnit: zapiUnitMap[name],
ZapiBaseCounter: zapiBaseCounterMap[name],
}
counters[harvestName] = co
}
Expand Down Expand Up @@ -326,7 +339,7 @@ func processZapiConfigCounters(path string) map[string]Counter {
return counters
}

func visitRestTemplates(dir string, client *Client, eachTemp func(path string, client *Client) map[string]Counter) map[string]Counter {
func visitRestTemplates(dir string, client *rest.Client, eachTemp func(path string, client *rest.Client) map[string]Counter) map[string]Counter {
result := make(map[string]Counter)
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
Expand Down Expand Up @@ -381,9 +394,9 @@ func updateDescription(description string) string {
return s
}

func generateCounterTemplate(counters map[string]Counter) {
targetPath := "docs/metrics.md"
t, err := template.New("counter.tmpl").ParseFiles("cmd/tools/rest/counter.tmpl")
func generateCounterTemplate(counters map[string]Counter, client *rest.Client) {
targetPath := "docs/ontap-metrics.md"
t, err := template.New("counter.tmpl").ParseFiles("cmd/tools/generate/counter.tmpl")
if err != nil {
panic(err)
}
Expand Down Expand Up @@ -420,6 +433,8 @@ func generateCounterTemplate(counters map[string]Counter) {

c := CounterTemplate{}
c.Counters = Values
verWithDots := strings.Trim(strings.Join(strings.Fields(fmt.Sprint(client.Cluster().Version)), "."), "[]")
c.CounterMetaData = CounterMetaData{Date: time.Now().Format("2006-Jan-02"), OntapVersion: verWithDots}

err = t.Execute(out, c)
if err != nil {
Expand All @@ -437,6 +452,7 @@ func mergeRestZapiCounters(restCounters map[string]Counter, zapiCounters map[str
v1.ZapiONTAPCounter = v.ZapiONTAPCounter
v1.ZapiUnit = v.ZapiUnit
v1.ZapiType = v.ZapiType
v1.ZapiBaseCounter = v.ZapiBaseCounter
restCounters[k] = v1
} else {
if v.ZapiONTAPCounter == "instance_name" || v.ZapiONTAPCounter == "instance_uuid" {
Expand All @@ -450,14 +466,15 @@ func mergeRestZapiCounters(restCounters map[string]Counter, zapiCounters map[str
ZapiONTAPCounter: v.ZapiONTAPCounter,
ZapiUnit: v.ZapiUnit,
ZapiType: v.ZapiType,
ZapiBaseCounter: v.ZapiBaseCounter,
}
restCounters[v.Name] = co
}
}
return restCounters
}

func processRestPerfCounters(path string, client *Client) map[string]Counter {
func processRestPerfCounters(path string, client *rest.Client) map[string]Counter {
var (
records []gjson.Result
counterSchema gjson.Result
Expand Down Expand Up @@ -486,8 +503,8 @@ func processRestPerfCounters(path string, client *Client) map[string]Counter {
}
}
}
href := BuildHref(query, "", nil, "", "", "", "", query)
records, err = Fetch(client, href)
href := rest.BuildHref(query, "", nil, "", "", "", "", query)
records, err = rest.Fetch(client, href)
if err != nil {
fmt.Printf("error while invoking api %+v\n", err)
return nil
Expand Down Expand Up @@ -521,6 +538,7 @@ func processRestPerfCounters(path string, client *Client) map[string]Counter {
Description: description,
RestType: ty,
RestUnit: r.Get("unit").String(),
RestBaseCounter: r.Get("denominator.name").String(),
}
counters[c.Name] = c
}
Expand All @@ -530,7 +548,7 @@ func processRestPerfCounters(path string, client *Client) map[string]Counter {
}

func ProcessExternalCounters(counters map[string]Counter) map[string]Counter {
dat, err := os.ReadFile("cmd/tools/rest/Counter.yaml")
dat, err := os.ReadFile("cmd/tools/generate/counter.yaml")
if err != nil {
fmt.Printf("error while reading file %v", err)
return nil
Expand Down
84 changes: 84 additions & 0 deletions cmd/tools/generate/counter.tmpl
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
This document contains the details about Harvest Metrics and their relevant ONTAP ZAPI and REST API mapping.
```
Creation Date : {{ .CounterMetaData.Date }}
ONTAP Version: {{ .CounterMetaData.OntapVersion }}
```
## Understanding the structure

Click on `Details` below to help understand the structure of [Metrics](#metrics).
<details>
<summary>Details</summary>

```yaml
- Harvest Metric: Name of metric exported by Harvest
Description: Description of the metric
ZAPI:
endpoint: Name of ZAPI invoked to generate this metric
metric: Metric name in ZAPI
template: ZAPI template path
unit: Unit of the counter. Possible values= per_sec, b_per_sec (bytes/s), kb_per_sec (Kbytes/s),mb_per_sec (Mbytes/s), percent, millisec, microsec, sec, or none
type: Comma separated list of properties of the counter. The counter properties determine how raw counter values should be interpreted.Possible values= raw, rate, delta, percent, string,no-display and no-zero-values.
base: Name of the counter used as the denominator to calculate values of counters involving averages and percentages.
REST:
endpoint: Name of REST API invoked to generate this metric
metric: Metric name in REST API
template: REST template path
unit: Unit of the counter. Possible values: per_sec, b_per_sec (bytes/s), kb_per_sec (Kbytes/s),mb_per_sec (Mbytes/s), percent, millisec, microsec, sec, or none
type: Comma separated list of properties of the counter. The counter properties determine how raw counter values should be interpreted.Possible values: raw, rate, delta, percent, string,no-display and no-zero-values.
base: Name of the counter used as the denominator to calculate values of counters involving averages and percentages.
```

</details>

## Metrics

```yaml
{{range .Counters}}
- Harvest Metric: {{ .Name }}
{{- if .Description }}
Description: {{ .Description }}
{{- end}}
{{- if .ZapiAPI }}
ZAPI:
{{- if .ZapiAPI }}
endpoint: {{ .ZapiAPI }}
{{- end}}
{{- if .ZapiONTAPCounter }}
metric: {{ .ZapiONTAPCounter }}
{{- end}}
{{- if .ZapiTemplate }}
template: {{ .ZapiTemplate }}
{{- end}}
{{- if .ZapiUnit }}
unit: {{ .ZapiUnit }}
{{- end}}
{{- if .ZapiType }}
type: {{ .ZapiType }}
{{- end}}
{{- if .ZapiBaseCounter }}
base: {{ .ZapiBaseCounter }}
{{- end}}
{{- end}}
{{- if .RestAPI }}
REST:
{{- if .RestAPI }}
endpoint: {{ .RestAPI }}
{{- end}}
{{- if .RestONTAPCounter }}
metric: {{ .RestONTAPCounter }}
{{- end}}
{{- if .RestTemplate }}
template: {{ .RestTemplate }}
{{- end}}
{{- if .RestUnit }}
unit: {{ .RestUnit }}
{{- end}}
{{- if .RestType }}
type: {{ .RestType }}
{{- end}}
{{- if .RestBaseCounter }}
base: {{ .RestBaseCounter }}
{{- end}}
{{- end}}
{{end}}
```
0 cmd/tools/rest/counter.yaml → cmd/tools/generate/counter.yaml
View file
File renamed without changes.
59 changes: 59 additions & 0 deletions cmd/tools/generate/generate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ package generate

import (
"fmt"
"github.com/netapp/harvest/v2/cmd/tools/rest"
"github.com/netapp/harvest/v2/pkg/api/ontapi/zapi"
"github.com/netapp/harvest/v2/pkg/color"
"github.com/netapp/harvest/v2/pkg/conf"
"github.com/spf13/cobra"
Expand All @@ -12,6 +14,7 @@ import (
"strconv"
"strings"
"text/template"
"time"
)

type PollerInfo struct {
Expand Down Expand Up @@ -49,6 +52,7 @@ type PromTemplate struct {
}

type options struct {
Poller string
loglevel int
image string
filesdPath string
Expand Down Expand Up @@ -89,6 +93,13 @@ var fullCmd = &cobra.Command{
Run: doDockerFull,
}

var metricCmd = &cobra.Command{
Use: "metrics",
Short: "generate Harvest metrics documentation",
Hidden: true,
Run: doGenerateMetrics,
}

func doDockerFull(cmd *cobra.Command, _ []string) {
var config = cmd.Root().PersistentFlags().Lookup("config")
generateFullCompose(conf.ConfigPath(config.Value.String()))
Expand All @@ -103,6 +114,11 @@ func doDockerCompose(cmd *cobra.Command, _ []string) {
generateDockerCompose(conf.ConfigPath(config.Value.String()))
}

func doGenerateMetrics(cmd *cobra.Command, _ []string) {
var config = cmd.Root().PersistentFlags().Lookup("config")
generateMetrics(conf.ConfigPath(config.Value.String()))
}

const (
full = 0
harvest = 1
Expand Down Expand Up @@ -325,14 +341,57 @@ func writeAdminSystemd(configFp string) {
println(color.Colorize("✓", color.Green) + " HTTP SD file: " + harvestAdminService + " created")
}

func generateMetrics(path string) {
var (
poller *conf.Poller
err error
restClient *rest.Client
zapiClient *zapi.Client
)

err = conf.LoadHarvestConfig(path)
if err != nil {
panic(err)
}

if poller, _, err = rest.GetPollerAndAddr(opts.Poller); err != nil {
return
}

timeout, _ := time.ParseDuration(rest.DefaultTimeout)
if restClient, err = rest.New(*poller, timeout); err != nil {
fmt.Printf("error creating new client %+v\n", err)
os.Exit(1)
}
if err = restClient.Init(2); err != nil {
fmt.Printf("error init rest client %+v\n", err)
os.Exit(1)
}

if zapiClient, err = zapi.New(*poller); err != nil {
fmt.Printf("error creating new client %+v\n", err)
os.Exit(1)
}

swaggerBytes = readSwaggerJSON()
restCounters := processRestCounters(restClient)
zapiCounters := processZapiCounters(zapiClient)
counters := mergeRestZapiCounters(restCounters, zapiCounters)
counters = ProcessExternalCounters(counters)
generateCounterTemplate(counters, restClient)
}

func init() {
Cmd.AddCommand(systemdCmd)
Cmd.AddCommand(metricCmd)
Cmd.AddCommand(dockerCmd)
dockerCmd.AddCommand(fullCmd)

dFlags := dockerCmd.PersistentFlags()
fFlags := fullCmd.PersistentFlags()

flags := metricCmd.PersistentFlags()
flags.StringVarP(&opts.Poller, "poller", "p", "", "name of poller (cluster), as defined in your harvest config")
dFlags.IntVarP(&opts.loglevel, "loglevel", "l", 2,
"logging level (0=trace, 1=debug, 2=info, 3=warning, 4=error, 5=critical)",
)
Expand Down
Loading

0 comments on commit 51b0d7c

Please sign in to comment.