Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Harvest should fetch certificates via a script #2238

Merged
merged 3 commits into from
Aug 1, 2023
Merged

feat: Harvest should fetch certificates via a script #2238

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1ac3f74
feat: Harvest should fetch certificates via a script
cgrinds Jul 26, 2023
7a1d84b
feat: Harvest should fetch certificates via a script
cgrinds Jul 26, 2023
901c371
feat: Harvest should fetch certificates via a script
cgrinds Jul 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 21 additions & 64 deletions cmd/collectors/storagegrid/rest/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package rest

import (
"bytes"
"crypto/tls"
"encoding/json"
"errors"
"fmt"
Expand All @@ -27,18 +26,17 @@ const (
)

type Client struct {
client *http.Client
request *http.Request
buffer *bytes.Buffer
Logger *logging.Logger
baseURL string
Cluster Cluster
username string
token string
Timeout time.Duration
logRest bool // used to log Rest request/response
APIPath string
auth *auth.Credentials
client *http.Client
request *http.Request
buffer *bytes.Buffer
Logger *logging.Logger
baseURL string
Cluster Cluster
token string
Timeout time.Duration
logRest bool // used to log Rest request/response
APIPath string
auth *auth.Credentials
}

type Cluster struct {
Expand Down Expand Up @@ -76,14 +74,12 @@ func NewClient(pollerName string, clientTimeout string, c *auth.Credentials) (*C

func New(poller *conf.Poller, timeout time.Duration, c *auth.Credentials) (*Client, error) {
var (
client Client
httpclient *http.Client
transport *http.Transport
cert tls.Certificate
addr string
href string
useInsecureTLS bool
err error
client Client
httpclient *http.Client
transport *http.Transport
addr string
href string
err error
)

client = Client{
Expand All @@ -100,49 +96,10 @@ func New(poller *conf.Poller, timeout time.Duration, c *auth.Credentials) (*Clie
client.baseURL = href
client.Timeout = timeout

// by default, enforce secure TLS, if not requested otherwise by user
if x := poller.UseInsecureTLS; x != nil {
useInsecureTLS = *poller.UseInsecureTLS
} else {
useInsecureTLS = false
}

pollerAuth, err := c.GetPollerAuth()
transport, err = c.Transport(nil)
if err != nil {
return nil, err
}
if pollerAuth.IsCert {
certPath := poller.SslCert
keyPath := poller.SslKey
if certPath == "" {
return nil, errs.New(errs.ErrMissingParam, "ssl_cert")
} else if keyPath == "" {
return nil, errs.New(errs.ErrMissingParam, "ssl_key")
} else if cert, err = tls.LoadX509KeyPair(certPath, keyPath); err != nil {
return nil, err
}

transport = &http.Transport{
Proxy: http.ProxyFromEnvironment,
TLSClientConfig: &tls.Config{
Certificates: []tls.Certificate{cert},
InsecureSkipVerify: useInsecureTLS}, //nolint:gosec
}
} else {
username := pollerAuth.Username
password := pollerAuth.Password
client.username = username
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we need to retain this line for username same as rest/client.go 89 line

if username == "" {
return nil, errs.New(errs.ErrMissingParam, "username")
} else if password == "" {
return nil, errs.New(errs.ErrMissingParam, "password")
}

transport = &http.Transport{
Proxy: http.ProxyFromEnvironment,
TLSClientConfig: &tls.Config{InsecureSkipVerify: useInsecureTLS}, //nolint:gosec
}
}

httpclient = &http.Client{Transport: transport, Timeout: timeout}
client.client = httpclient
Expand Down Expand Up @@ -380,13 +337,13 @@ func (c *Client) fetchTokenWithAuthRetry() error {
if err != nil {
return fmt.Errorf("failed to create auth URL err: %w", err)
}
password, err := c.auth.Password()
pollerAuth, err := c.auth.GetPollerAuth()
if err != nil {
return err
}
authB := authBody{
Username: c.username,
Password: password,
Username: pollerAuth.Username,
Password: pollerAuth.Password,
}
postBody, err := json.Marshal(authB)
if err != nil {
Expand Down
31 changes: 1 addition & 30 deletions cmd/poller/poller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ import (
"github.com/netapp/harvest/v2/pkg/requests"
"github.com/netapp/harvest/v2/pkg/tree/node"
"github.com/netapp/harvest/v2/pkg/util"
"github.com/rs/zerolog/log"
"github.com/spf13/cobra"
"gopkg.in/yaml.v3"
"io"
Expand All @@ -64,7 +63,6 @@ import (
"os"
"os/exec"
"os/signal"
"path"
"regexp"
"runtime"
"strconv"
Expand Down Expand Up @@ -243,33 +241,6 @@ func (p *Poller) Init() error {

// create a shared auth service that all collectors will use
p.auth = auth.NewCredentials(p.params, logger)
pollerAuth, err := p.auth.GetPollerAuth()
if err != nil {
return err
}

// check optional parameter auth_style
// if certificates are missing use default paths
if pollerAuth.IsCert {
if p.params.SslCert == "" {
fp := path.Join(p.options.HomePath, "cert/", p.options.Hostname+".pem")
p.params.SslCert = fp
logger.Debug().Msgf("using default [ssl_cert] path: [%s]", fp)
if _, err = os.Stat(fp); err != nil {
logger.Error().Stack().Err(err).Msgf("ssl_cert")
return errs.New(errs.ErrMissingParam, "ssl_cert: "+err.Error())
}
}
if p.params.SslKey == "" {
fp := path.Join(p.options.HomePath, "cert/", p.options.Hostname+".key")
p.params.SslKey = fp
logger.Debug().Msgf("using default [ssl_key] path: [%s]", fp)
if _, err = os.Stat(fp); err != nil {
logger.Error().Stack().Err(err).Msgf("ssl_key")
return errs.New(errs.ErrMissingParam, "ssl_key: "+err.Error())
}
}
}

// initialize our metadata, the metadata will host status of our
// collectors and exporters, as well as ping stats to target host
Expand Down Expand Up @@ -1167,7 +1138,7 @@ func (p *Poller) negotiateAPI(c conf.Collector, checkZAPIs func() error) conf.Co
Templates: c.Templates,
}
}
log.Error().Err(err).Str("collector", c.Name).Msg("Failed to negotiateAPI")
logger.Error().Err(err).Str("collector", c.Name).Msg("Failed to negotiateAPI")
}

return c
Expand Down
133 changes: 31 additions & 102 deletions cmd/tools/rest/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@ package rest

import (
"bytes"
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"github.com/netapp/harvest/v2/pkg/auth"
Expand Down Expand Up @@ -33,16 +31,15 @@ const (
)

type Client struct {
client *http.Client
request *http.Request
buffer *bytes.Buffer
Logger *logging.Logger
baseURL string
cluster Cluster
username string
Timeout time.Duration
logRest bool // used to log Rest request/response
auth *auth.Credentials
client *http.Client
request *http.Request
buffer *bytes.Buffer
Logger *logging.Logger
baseURL string
cluster Cluster
Timeout time.Duration
logRest bool // used to log Rest request/response
auth *auth.Credentials
}

type Cluster struct {
Expand All @@ -54,14 +51,12 @@ type Cluster struct {

func New(poller *conf.Poller, timeout time.Duration, auth *auth.Credentials) (*Client, error) {
var (
client Client
httpclient *http.Client
transport *http.Transport
cert tls.Certificate
addr string
url string
useInsecureTLS bool
err error
client Client
httpclient *http.Client
transport *http.Transport
addr string
url string
err error
)

client = Client{
Expand All @@ -81,69 +76,10 @@ func New(poller *conf.Poller, timeout time.Duration, auth *auth.Credentials) (*C
client.baseURL = url
client.Timeout = timeout

// by default, enforce secure TLS, if not requested otherwise by user
if x := poller.UseInsecureTLS; x != nil {
useInsecureTLS = *poller.UseInsecureTLS
} else {
useInsecureTLS = false
}

pollerAuth, err := auth.GetPollerAuth()
transport, err = auth.Transport(nil)
if err != nil {
return nil, err
}

if pollerAuth.IsCert {
sslCertPath := poller.SslCert
keyPath := poller.SslKey
caCertPath := poller.CaCertPath

if sslCertPath == "" {
return nil, errs.New(errs.ErrMissingParam, "ssl_cert")
} else if keyPath == "" {
return nil, errs.New(errs.ErrMissingParam, "ssl_key")
} else if cert, err = tls.LoadX509KeyPair(sslCertPath, keyPath); err != nil {
return nil, err
}

// Create a CA certificate pool and add certificate if specified
caCertPool := x509.NewCertPool()
if caCertPath != "" {
caCert, err := os.ReadFile(caCertPath)
if err != nil {
client.Logger.Error().Err(err).Str("cacert", caCertPath).Msg("Failed to read ca cert")
// continue
}
if caCert != nil {
pem := caCertPool.AppendCertsFromPEM(caCert)
if !pem {
client.Logger.Error().Err(err).Str("cacert", caCertPath).Msg("Failed to append ca cert")
// continue
}
}
}

transport = &http.Transport{
Proxy: http.ProxyFromEnvironment,
TLSClientConfig: &tls.Config{
RootCAs: caCertPool,
Certificates: []tls.Certificate{cert},
InsecureSkipVerify: useInsecureTLS}, //nolint:gosec
}
} else {
if pollerAuth.Username == "" {
return nil, errs.New(errs.ErrMissingParam, "username")
} else if pollerAuth.Password == "" {
return nil, errs.New(errs.ErrMissingParam, "password")
}
client.username = pollerAuth.Username

transport = &http.Transport{
Proxy: http.ProxyFromEnvironment,
TLSClientConfig: &tls.Config{InsecureSkipVerify: useInsecureTLS}, //nolint:gosec
}
}

transport.DialContext = (&net.Dialer{Timeout: DefaultDialerTimeout}).DialContext
httpclient = &http.Client{Transport: transport, Timeout: timeout}
client.client = httpclient
Expand Down Expand Up @@ -191,12 +127,12 @@ func (c *Client) GetRest(request string) ([]byte, error) {
return nil, err
}
c.request.Header.Set("accept", "application/json")
if c.username != "" {
password, err2 := c.auth.Password()
if err2 != nil {
return nil, err2
}
c.request.SetBasicAuth(c.username, password)
pollerAuth, err := c.auth.GetPollerAuth()
if err != nil {
return nil, err
}
if pollerAuth.Username != "" {
c.request.SetBasicAuth(pollerAuth.Username, pollerAuth.Password)
}
// ensure that we can change body dynamically
c.request.GetBody = func() (io.ReadCloser, error) {
Expand Down Expand Up @@ -288,11 +224,11 @@ func (c *Client) invokeWithAuthRetry() ([]byte, error) {
}
if pollerAuth.HasCredentialScript {
c.auth.Expire()
password, err2 := c.auth.Password()
pollerAuth2, err2 := c.auth.GetPollerAuth()
if err2 != nil {
return nil, err2
}
c.request.SetBasicAuth(pollerAuth.Username, password)
c.request.SetBasicAuth(pollerAuth2.Username, pollerAuth2.Password)
return doInvoke()
}
}
Expand All @@ -302,8 +238,6 @@ func (c *Client) invokeWithAuthRetry() ([]byte, error) {
}

func downloadSwagger(poller *conf.Poller, path string, url string, verbose bool) (int64, error) {
var restClient *Client

out, err := os.Create(path)
if err != nil {
return 0, fmt.Errorf("unable to create %s to save swagger.yaml", path)
Expand All @@ -315,23 +249,18 @@ func downloadSwagger(poller *conf.Poller, path string, url string, verbose bool)
}

timeout, _ := time.ParseDuration(DefaultTimeout)
if restClient, err = New(poller, timeout, auth.NewCredentials(poller, logging.Get())); err != nil {
return 0, fmt.Errorf("error creating new client %w", err)
credentials := auth.NewCredentials(poller, logging.Get())
transport, err := credentials.Transport(request)
if err != nil {
return 0, err
}
httpclient := &http.Client{Transport: transport, Timeout: timeout}

downClient := &http.Client{Transport: restClient.client.Transport, Timeout: restClient.client.Timeout}
if restClient.username != "" {
password, err2 := restClient.auth.Password()
if err2 != nil {
return 0, err2
}
request.SetBasicAuth(restClient.username, password)
}
if verbose {
requestOut, _ := httputil.DumpRequestOut(request, false)
fmt.Printf("REQUEST: %s BY: %s\n%s\n", url, restClient.username, requestOut)
fmt.Printf("REQUEST: %s\n%s\n", url, requestOut)
}
response, err := downClient.Do(request)
response, err := httpclient.Do(request)
if err != nil {
return 0, err
}
Expand Down
6 changes: 5 additions & 1 deletion cmd/tools/rest/rest.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -319,7 +319,11 @@ func FetchForCli(client *Client, href string, records *[]any, downloadAll bool,
return fmt.Errorf("error making request %w", err)
}

*curls = append(*curls, fmt.Sprintf("curl --user %s --insecure '%s%s'", client.username, client.baseURL, href))
pollerAuth, err := client.auth.GetPollerAuth()
if err != nil {
return err
}
*curls = append(*curls, fmt.Sprintf("curl --user %s --insecure '%s%s'", pollerAuth.Username, client.baseURL, href))

isNonIterRestCall := false
value := gjson.GetBytes(getRest, "records")
Expand Down
Loading