Skip to content

Commit

Permalink
refactor: remove openapi dependency (#3436)
Browse files Browse the repository at this point in the history 
suggest ONTAP REST API Online Reference
  • Loading branch information
@cgrinds
cgrinds authored Jan 27, 2025
1 parent 8a798a1 commit c7eed53
Show file tree
Hide file tree
Showing 95 changed files with 135 additions and 15,642 deletions.
98 changes: 97 additions & 1 deletion cmd/tools/generate/counter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,17 +9,23 @@ import (
"github.com/netapp/harvest/v2/cmd/tools/rest"
template2 "github.com/netapp/harvest/v2/cmd/tools/template"
"github.com/netapp/harvest/v2/pkg/api/ontapi/zapi"
"github.com/netapp/harvest/v2/pkg/auth"
"github.com/netapp/harvest/v2/pkg/conf"
"github.com/netapp/harvest/v2/pkg/logging"
"github.com/netapp/harvest/v2/pkg/requests"
"github.com/netapp/harvest/v2/pkg/set"
"github.com/netapp/harvest/v2/pkg/tree"
"github.com/netapp/harvest/v2/pkg/tree/node"
"github.com/netapp/harvest/v2/pkg/util"
tw "github.com/netapp/harvest/v2/third_party/olekukonko/tablewriter"
"github.com/netapp/harvest/v2/third_party/tidwall/gjson"
"gopkg.in/yaml.v3"
"io"
"log"
"log/slog"
"maps"
"net/http"
"net/http/httputil"
"net/url"
"os"
"os/exec"
Expand Down Expand Up @@ -247,7 +253,7 @@ func (c Counter) HasAPIs() bool {
// readSwaggerJSON downloads poller swagger and convert to json format
func readSwaggerJSON() []byte {
var f []byte
path, err := rest.ReadOrDownloadSwagger(opts.Poller)
path, err := downloadSwaggerForPoller(opts.Poller)
if err != nil {
log.Fatal("failed to download swagger:", err)
return nil
Expand All @@ -260,6 +266,96 @@ func readSwaggerJSON() []byte {
}
return f
}
func downloadSwaggerForPoller(pName string) (string, error) {
var (
poller *conf.Poller
err error
addr string
shouldDownload = true
swagTime time.Time
)

if poller, addr, err = rest.GetPollerAndAddr(pName); err != nil {
return "", err
}

tmp := os.TempDir()
swaggerPath := filepath.Join(tmp, addr+"-swagger.yaml")
fileInfo, err := os.Stat(swaggerPath)

if os.IsNotExist(err) {
fmt.Printf("%s does not exist downloading\n", swaggerPath)
} else {
swagTime = fileInfo.ModTime()
twoWeeksAgo := swagTime.Local().AddDate(0, 0, -14)
if swagTime.Before(twoWeeksAgo) {
fmt.Printf("%s is more than two weeks old, re-download", swaggerPath)
} else {
shouldDownload = false
}
}

if shouldDownload {
swaggerURL := "https://" + addr + "/docs/api/swagger.yaml"
bytesDownloaded, err := downloadSwagger(poller, swaggerPath, swaggerURL, false)
if err != nil {
fmt.Printf("error downloading swagger %s\n", err)
if bytesDownloaded == 0 {
// if the tmp file exists, remove it since it is empty
_ = os.Remove(swaggerPath)
}
return "", err
}
fmt.Printf("downloaded %d bytes from %s\n", bytesDownloaded, swaggerURL)
}

fmt.Printf("Using downloaded file %s with timestamp %s\n", swaggerPath, swagTime)
return swaggerPath, nil
}

func downloadSwagger(poller *conf.Poller, path string, url string, verbose bool) (int64, error) {
out, err := os.Create(path)
if err != nil {
return 0, fmt.Errorf("unable to create %s to save swagger.yaml", path)
}
defer func(out *os.File) { _ = out.Close() }(out)
request, err := requests.New("GET", url, nil)
if err != nil {
return 0, err
}

timeout, _ := time.ParseDuration(rest.DefaultTimeout)
credentials := auth.NewCredentials(poller, slog.Default())
transport, err := credentials.Transport(request, poller)
if err != nil {
return 0, err
}
httpclient := &http.Client{Transport: transport, Timeout: timeout}

if verbose {
requestOut, _ := httputil.DumpRequestOut(request, false)
fmt.Printf("REQUEST: %s\n%s\n", url, requestOut)
}
response, err := httpclient.Do(request)
if err != nil {
return 0, err
}
//goland:noinspection GoUnhandledErrorResult
defer response.Body.Close()

if verbose {
debugResp, _ := httputil.DumpResponse(response, false)
fmt.Printf("RESPONSE: \n%s", debugResp)
}
if response.StatusCode != http.StatusOK {
return 0, fmt.Errorf("error making request. server response statusCode=[%d]", response.StatusCode)
}
n, err := io.Copy(out, response.Body)
if err != nil {
return 0, fmt.Errorf("error while downloading %s err=%w", url, err)
}
return n, nil
}

// searchDescriptionSwagger returns ontap counter description from swagger
func searchDescriptionSwagger(objName string, ontapCounterName string) string {
Expand Down
46 changes: 0 additions & 46 deletions cmd/tools/rest/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,6 @@ import (
"io"
"log/slog"
"net/http"
"net/http/httputil"
"os"
"slices"
"strings"
"time"
Expand Down Expand Up @@ -272,50 +270,6 @@ func (c *Client) invokeWithAuthRetry() ([]byte, error) {
return body, err
}

func downloadSwagger(poller *conf.Poller, path string, url string, verbose bool) (int64, error) {
out, err := os.Create(path)
if err != nil {
return 0, fmt.Errorf("unable to create %s to save swagger.yaml", path)
}
defer func(out *os.File) { _ = out.Close() }(out)
request, err := requests.New("GET", url, nil)
if err != nil {
return 0, err
}

timeout, _ := time.ParseDuration(DefaultTimeout)
credentials := auth.NewCredentials(poller, slog.Default())
transport, err := credentials.Transport(request, poller)
if err != nil {
return 0, err
}
httpclient := &http.Client{Transport: transport, Timeout: timeout}

if verbose {
requestOut, _ := httputil.DumpRequestOut(request, false)
fmt.Printf("REQUEST: %s\n%s\n", url, requestOut)
}
response, err := httpclient.Do(request)
if err != nil {
return 0, err
}
//goland:noinspection GoUnhandledErrorResult
defer response.Body.Close()

if verbose {
debugResp, _ := httputil.DumpResponse(response, false)
fmt.Printf("RESPONSE: \n%s", debugResp)
}
if response.StatusCode != http.StatusOK {
return 0, fmt.Errorf("error making request. server response statusCode=[%d]", response.StatusCode)
}
n, err := io.Copy(out, response.Body)
if err != nil {
return 0, fmt.Errorf("error while downloading %s err=%w", url, err)
}
return n, nil
}

func (c *Client) UpdateClusterInfo(retries int) error {
var (
err error
Expand Down
97 changes: 21 additions & 76 deletions cmd/tools/rest/rest.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ import (
"net/url"
"os"
"os/signal"
"path/filepath"
"strconv"
"strings"
"time"
Expand All @@ -35,21 +34,18 @@ type check struct {
var args = &Args{}

type Args struct {
Item string
Poller string
API string
Endpoint string
Config string
SwaggerPath string
Fields string
Field []string
QueryField string
QueryValue string
DownloadAll bool
MaxRecords string
ForceDownload bool
Verbose bool
Timeout string
Item string
Poller string
API string
Endpoint string
Config string
Fields string
Field []string
QueryField string
QueryValue string
DownloadAll bool
MaxRecords string
Timeout string
}

var Cmd = &cobra.Command{
Expand All @@ -66,49 +62,17 @@ var showCmd = &cobra.Command{
Run: doShow,
}

func ReadOrDownloadSwagger(pName string) (string, error) {
func OntapRestAPIHref(pName string) (string, error) {
var (
poller *conf.Poller
err error
addr string
shouldDownload = true
swagTime time.Time
err error
addr string
)

if poller, addr, err = GetPollerAndAddr(pName); err != nil {
if _, addr, err = GetPollerAndAddr(pName); err != nil {
return "", err
}

tmp := os.TempDir()
swaggerPath := filepath.Join(tmp, addr+"-swagger.yaml")
fileInfo, err := os.Stat(swaggerPath)

if os.IsNotExist(err) {
fmt.Printf("%s does not exist downloading\n", swaggerPath)
} else if !args.ForceDownload {
swagTime = fileInfo.ModTime()
twoWeeksAgo := swagTime.Local().AddDate(0, 0, -14)
if swagTime.Before(twoWeeksAgo) {
fmt.Printf("%s is more than two weeks old, re-download", swaggerPath)
} else {
shouldDownload = false
}
}
if shouldDownload {
swaggerURL := "https://" + addr + "/docs/api/swagger.yaml"
bytesDownloaded, err := downloadSwagger(poller, swaggerPath, swaggerURL, args.Verbose)
if err != nil {
fmt.Printf("error downloading swagger %s\n", err)
if bytesDownloaded == 0 {
// if the tmp file exists, remove it since it is empty
_ = os.Remove(swaggerPath)
}
return "", err
}
fmt.Printf("downloaded %d bytes from %s\n", bytesDownloaded, swaggerURL)
}
fmt.Printf("Using downloaded file %s with timestamp %s\n", swaggerPath, swagTime)
return swaggerPath, nil
return "https://" + addr + "/docs/api/", nil
}

func doShow(_ *cobra.Command, a []string) {
Expand All @@ -120,19 +84,10 @@ func doShow(_ *cobra.Command, a []string) {
if err != nil {
log.Fatal(err)
}
if args.SwaggerPath != "" {
doSwagger(*args)
} else {
doCmd()
}
doCmd()
}

func validateArgs(slice []string) check {
// One of Poller or SwaggerPath are allowed, but not both
if args.Poller != "" && args.SwaggerPath != "" {
fmt.Printf("Both poller and swagger are set. Only one or the other can be set, not both\n")
return check{isValid: false}
}
if len(slice) == 0 {
args.Item = ""
} else {
Expand All @@ -150,13 +105,12 @@ func validateArgs(slice []string) check {
func doCmd() {
switch args.Item {
case "apis", "params", "models":
swaggerPath, err := ReadOrDownloadSwagger(args.Poller)
ontapRestAPI, err := OntapRestAPIHref(args.Poller)
if err != nil {
fmt.Printf("error %+v\n", err)
return
}
args.SwaggerPath = swaggerPath
doSwagger(*args)
fmt.Printf("Find the ONTAP REST API reference for %s here: %s\n", args.Poller, ontapRestAPI)
case "data":
doData()
}
Expand Down Expand Up @@ -750,16 +704,13 @@ func init() {
Cmd.AddCommand(showCmd)
flags := Cmd.PersistentFlags()
flags.StringVarP(&args.Poller, "poller", "p", "", "Name of poller (cluster), as defined in your harvest config. * for all pollers")
flags.StringVarP(&args.SwaggerPath, "swagger", "s", "", "Path to Swagger (OpenAPI) file to read from")
flags.StringVar(&args.Config, "config", configPath, "Harvest config file path")
flags.StringVarP(&args.Timeout, "timeout", "t", DefaultTimeout, "Duration to wait before giving up")

showFlags := showCmd.Flags()
showFlags.StringVarP(&args.API, "api", "a", "", "REST API PATTERN to show")
showFlags.BoolVar(&args.DownloadAll, "all", false, "Collect all records by walking pagination links")
showFlags.BoolVarP(&args.Verbose, "verbose", "v", false, "Be verbose")
showFlags.StringVarP(&args.MaxRecords, "max-records", "m", "", "Limit the number of records returned before providing pagination link")
showFlags.BoolVar(&args.ForceDownload, "download", false, "Force download Swagger file instead of using local copy")
showFlags.StringVarP(&args.Fields, "fields", "f", "*", "Fields to return in the response <field>[,...].")
showFlags.StringArrayVar(&args.Field, "field", []string{}, "Query a field by value (can be specified multiple times.)\n"+
`If the value contains query characters (*|,!<>..), it must be quoted to avoid their special meaning
Expand All @@ -776,15 +727,9 @@ func init() {

Cmd.SetUsageTemplate(Cmd.UsageTemplate() + `
Examples:
# Query cluster infinity for available APIs
# Print cluster infinity's' ONTAP REST API Online Reference
bin/harvest rest -p infinity show apis
# Query cluster infinity for svm parameters. These query parameters are used to filter requests.
bin/harvest rest -p infinity show params --api svm/svms
# Query cluster infinity for svm models. These describe the REST response of sending the svm/svms GET request.
bin/harvest rest -p infinity show models --api svm/svms
# Query cluster infinity for stopped svms.
bin/harvest rest -p infinity show data --api svm/svms --field "state=stopped"
Expand Down
Loading

0 comments on commit c7eed53

Please sign in to comment.