Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: remove openapi dependency #3436

Merged
merged 1 commit into from
Jan 27, 2025
Merged

refactor: remove openapi dependency #3436

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 97 additions & 1 deletion cmd/tools/generate/counter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,17 +9,23 @@ import (
"github.com/netapp/harvest/v2/cmd/tools/rest"
template2 "github.com/netapp/harvest/v2/cmd/tools/template"
"github.com/netapp/harvest/v2/pkg/api/ontapi/zapi"
"github.com/netapp/harvest/v2/pkg/auth"
"github.com/netapp/harvest/v2/pkg/conf"
"github.com/netapp/harvest/v2/pkg/logging"
"github.com/netapp/harvest/v2/pkg/requests"
"github.com/netapp/harvest/v2/pkg/set"
"github.com/netapp/harvest/v2/pkg/tree"
"github.com/netapp/harvest/v2/pkg/tree/node"
"github.com/netapp/harvest/v2/pkg/util"
tw "github.com/netapp/harvest/v2/third_party/olekukonko/tablewriter"
"github.com/netapp/harvest/v2/third_party/tidwall/gjson"
"gopkg.in/yaml.v3"
"io"
"log"
"log/slog"
"maps"
"net/http"
"net/http/httputil"
"net/url"
"os"
"os/exec"
Expand Down Expand Up @@ -247,7 +253,7 @@ func (c Counter) HasAPIs() bool {
// readSwaggerJSON downloads poller swagger and convert to json format
func readSwaggerJSON() []byte {
var f []byte
path, err := rest.ReadOrDownloadSwagger(opts.Poller)
path, err := downloadSwaggerForPoller(opts.Poller)
if err != nil {
log.Fatal("failed to download swagger:", err)
return nil
Expand All @@ -260,6 +266,96 @@ func readSwaggerJSON() []byte {
}
return f
}
func downloadSwaggerForPoller(pName string) (string, error) {
var (
poller *conf.Poller
err error
addr string
shouldDownload = true
swagTime time.Time
)

if poller, addr, err = rest.GetPollerAndAddr(pName); err != nil {
return "", err
}

tmp := os.TempDir()
swaggerPath := filepath.Join(tmp, addr+"-swagger.yaml")
fileInfo, err := os.Stat(swaggerPath)

if os.IsNotExist(err) {
fmt.Printf("%s does not exist downloading\n", swaggerPath)
} else {
swagTime = fileInfo.ModTime()
twoWeeksAgo := swagTime.Local().AddDate(0, 0, -14)
if swagTime.Before(twoWeeksAgo) {
fmt.Printf("%s is more than two weeks old, re-download", swaggerPath)
} else {
shouldDownload = false
}
}

if shouldDownload {
swaggerURL := "https://" + addr + "/docs/api/swagger.yaml"
bytesDownloaded, err := downloadSwagger(poller, swaggerPath, swaggerURL, false)
if err != nil {
fmt.Printf("error downloading swagger %s\n", err)
if bytesDownloaded == 0 {
// if the tmp file exists, remove it since it is empty
_ = os.Remove(swaggerPath)
}
return "", err
}
fmt.Printf("downloaded %d bytes from %s\n", bytesDownloaded, swaggerURL)
}

fmt.Printf("Using downloaded file %s with timestamp %s\n", swaggerPath, swagTime)
return swaggerPath, nil
}

func downloadSwagger(poller *conf.Poller, path string, url string, verbose bool) (int64, error) {
out, err := os.Create(path)
if err != nil {
return 0, fmt.Errorf("unable to create %s to save swagger.yaml", path)
}
defer func(out *os.File) { _ = out.Close() }(out)
request, err := requests.New("GET", url, nil)
if err != nil {
return 0, err
}

timeout, _ := time.ParseDuration(rest.DefaultTimeout)
credentials := auth.NewCredentials(poller, slog.Default())
transport, err := credentials.Transport(request, poller)
if err != nil {
return 0, err
}
httpclient := &http.Client{Transport: transport, Timeout: timeout}

if verbose {
requestOut, _ := httputil.DumpRequestOut(request, false)
fmt.Printf("REQUEST: %s\n%s\n", url, requestOut)
}
response, err := httpclient.Do(request)
if err != nil {
return 0, err
}
//goland:noinspection GoUnhandledErrorResult
defer response.Body.Close()

if verbose {
debugResp, _ := httputil.DumpResponse(response, false)
fmt.Printf("RESPONSE: \n%s", debugResp)
}
if response.StatusCode != http.StatusOK {
return 0, fmt.Errorf("error making request. server response statusCode=[%d]", response.StatusCode)
}
n, err := io.Copy(out, response.Body)
if err != nil {
return 0, fmt.Errorf("error while downloading %s err=%w", url, err)
}
return n, nil
}

// searchDescriptionSwagger returns ontap counter description from swagger
func searchDescriptionSwagger(objName string, ontapCounterName string) string {
Expand Down
46 changes: 0 additions & 46 deletions cmd/tools/rest/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,6 @@ import (
"io"
"log/slog"
"net/http"
"net/http/httputil"
"os"
"slices"
"strings"
"time"
Expand Down Expand Up @@ -272,50 +270,6 @@ func (c *Client) invokeWithAuthRetry() ([]byte, error) {
return body, err
}

func downloadSwagger(poller *conf.Poller, path string, url string, verbose bool) (int64, error) {
out, err := os.Create(path)
if err != nil {
return 0, fmt.Errorf("unable to create %s to save swagger.yaml", path)
}
defer func(out *os.File) { _ = out.Close() }(out)
request, err := requests.New("GET", url, nil)
if err != nil {
return 0, err
}

timeout, _ := time.ParseDuration(DefaultTimeout)
credentials := auth.NewCredentials(poller, slog.Default())
transport, err := credentials.Transport(request, poller)
if err != nil {
return 0, err
}
httpclient := &http.Client{Transport: transport, Timeout: timeout}

if verbose {
requestOut, _ := httputil.DumpRequestOut(request, false)
fmt.Printf("REQUEST: %s\n%s\n", url, requestOut)
}
response, err := httpclient.Do(request)
if err != nil {
return 0, err
}
//goland:noinspection GoUnhandledErrorResult
defer response.Body.Close()

if verbose {
debugResp, _ := httputil.DumpResponse(response, false)
fmt.Printf("RESPONSE: \n%s", debugResp)
}
if response.StatusCode != http.StatusOK {
return 0, fmt.Errorf("error making request. server response statusCode=[%d]", response.StatusCode)
}
n, err := io.Copy(out, response.Body)
if err != nil {
return 0, fmt.Errorf("error while downloading %s err=%w", url, err)
}
return n, nil
}

func (c *Client) UpdateClusterInfo(retries int) error {
var (
err error
Expand Down
97 changes: 21 additions & 76 deletions cmd/tools/rest/rest.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ import (
"net/url"
"os"
"os/signal"
"path/filepath"
"strconv"
"strings"
"time"
Expand All @@ -35,21 +34,18 @@ type check struct {
var args = &Args{}

type Args struct {
Item string
Poller string
API string
Endpoint string
Config string
SwaggerPath string
Fields string
Field []string
QueryField string
QueryValue string
DownloadAll bool
MaxRecords string
ForceDownload bool
Verbose bool
Timeout string
Item string
Poller string
API string
Endpoint string
Config string
Fields string
Field []string
QueryField string
QueryValue string
DownloadAll bool
MaxRecords string
Timeout string
}

var Cmd = &cobra.Command{
Expand All @@ -66,49 +62,17 @@ var showCmd = &cobra.Command{
Run: doShow,
}

func ReadOrDownloadSwagger(pName string) (string, error) {
func OntapRestAPIHref(pName string) (string, error) {
var (
poller *conf.Poller
err error
addr string
shouldDownload = true
swagTime time.Time
err error
addr string
)

if poller, addr, err = GetPollerAndAddr(pName); err != nil {
if _, addr, err = GetPollerAndAddr(pName); err != nil {
return "", err
}

tmp := os.TempDir()
swaggerPath := filepath.Join(tmp, addr+"-swagger.yaml")
fileInfo, err := os.Stat(swaggerPath)

if os.IsNotExist(err) {
fmt.Printf("%s does not exist downloading\n", swaggerPath)
} else if !args.ForceDownload {
swagTime = fileInfo.ModTime()
twoWeeksAgo := swagTime.Local().AddDate(0, 0, -14)
if swagTime.Before(twoWeeksAgo) {
fmt.Printf("%s is more than two weeks old, re-download", swaggerPath)
} else {
shouldDownload = false
}
}
if shouldDownload {
swaggerURL := "https://" + addr + "/docs/api/swagger.yaml"
bytesDownloaded, err := downloadSwagger(poller, swaggerPath, swaggerURL, args.Verbose)
if err != nil {
fmt.Printf("error downloading swagger %s\n", err)
if bytesDownloaded == 0 {
// if the tmp file exists, remove it since it is empty
_ = os.Remove(swaggerPath)
}
return "", err
}
fmt.Printf("downloaded %d bytes from %s\n", bytesDownloaded, swaggerURL)
}
fmt.Printf("Using downloaded file %s with timestamp %s\n", swaggerPath, swagTime)
return swaggerPath, nil
return "https://" + addr + "/docs/api/", nil
}

func doShow(_ *cobra.Command, a []string) {
Expand All @@ -120,19 +84,10 @@ func doShow(_ *cobra.Command, a []string) {
if err != nil {
log.Fatal(err)
}
if args.SwaggerPath != "" {
doSwagger(*args)
} else {
doCmd()
}
doCmd()
}

func validateArgs(slice []string) check {
// One of Poller or SwaggerPath are allowed, but not both
if args.Poller != "" && args.SwaggerPath != "" {
fmt.Printf("Both poller and swagger are set. Only one or the other can be set, not both\n")
return check{isValid: false}
}
if len(slice) == 0 {
args.Item = ""
} else {
Expand All @@ -150,13 +105,12 @@ func validateArgs(slice []string) check {
func doCmd() {
switch args.Item {
case "apis", "params", "models":
swaggerPath, err := ReadOrDownloadSwagger(args.Poller)
ontapRestAPI, err := OntapRestAPIHref(args.Poller)
if err != nil {
fmt.Printf("error %+v\n", err)
return
}
args.SwaggerPath = swaggerPath
doSwagger(*args)
fmt.Printf("Find the ONTAP REST API reference for %s here: %s\n", args.Poller, ontapRestAPI)
case "data":
doData()
}
Expand Down Expand Up @@ -750,16 +704,13 @@ func init() {
Cmd.AddCommand(showCmd)
flags := Cmd.PersistentFlags()
flags.StringVarP(&args.Poller, "poller", "p", "", "Name of poller (cluster), as defined in your harvest config. * for all pollers")
flags.StringVarP(&args.SwaggerPath, "swagger", "s", "", "Path to Swagger (OpenAPI) file to read from")
flags.StringVar(&args.Config, "config", configPath, "Harvest config file path")
flags.StringVarP(&args.Timeout, "timeout", "t", DefaultTimeout, "Duration to wait before giving up")

showFlags := showCmd.Flags()
showFlags.StringVarP(&args.API, "api", "a", "", "REST API PATTERN to show")
showFlags.BoolVar(&args.DownloadAll, "all", false, "Collect all records by walking pagination links")
showFlags.BoolVarP(&args.Verbose, "verbose", "v", false, "Be verbose")
showFlags.StringVarP(&args.MaxRecords, "max-records", "m", "", "Limit the number of records returned before providing pagination link")
showFlags.BoolVar(&args.ForceDownload, "download", false, "Force download Swagger file instead of using local copy")
showFlags.StringVarP(&args.Fields, "fields", "f", "*", "Fields to return in the response <field>[,...].")
showFlags.StringArrayVar(&args.Field, "field", []string{}, "Query a field by value (can be specified multiple times.)\n"+
`If the value contains query characters (*|,!<>..), it must be quoted to avoid their special meaning
Expand All @@ -776,15 +727,9 @@ func init() {

Cmd.SetUsageTemplate(Cmd.UsageTemplate() + `
Examples:
# Query cluster infinity for available APIs
# Print cluster infinity's' ONTAP REST API Online Reference
bin/harvest rest -p infinity show apis

# Query cluster infinity for svm parameters. These query parameters are used to filter requests.
bin/harvest rest -p infinity show params --api svm/svms

# Query cluster infinity for svm models. These describe the REST response of sending the svm/svms GET request.
bin/harvest rest -p infinity show models --api svm/svms

# Query cluster infinity for stopped svms.
bin/harvest rest -p infinity show data --api svm/svms --field "state=stopped"

Expand Down
Loading
Loading