From 4bf412d5380283497688f9a9e4e82de04d9160d1 Mon Sep 17 00:00:00 2001 From: "Ryan J. Geyer" Date: Thu, 22 Jul 2021 14:56:37 -0700 Subject: [PATCH] Refactor config to allow it to be set externally when used as a library --- config/config.go | 191 ++++++++++++++++++++++++++++------------- exporter/gather.go | 14 +-- exporter/prometheus.go | 2 +- 3 files changed, 137 insertions(+), 70 deletions(-) diff --git a/config/config.go b/config/config.go index bdb1dd04..e84c66dc 100644 --- a/config/config.go +++ b/config/config.go @@ -1,8 +1,9 @@ package config import ( - "fmt" "io/ioutil" + "net/url" + "path" "strings" log "github.com/sirupsen/logrus" @@ -15,14 +16,12 @@ import ( // Config struct holds all of the runtime confgiguration for the application type Config struct { *cfg.BaseConfig - APIURL string - Repositories string - Organisations string - Users string - APITokenEnv string - APITokenFile string - APIToken string - TargetURLs []string + apiUrl *url.URL + repositories []string + organisations []string + users []string + apiToken string + targetURLs []string } // Init populates the Config struct based on environmental runtime configuration @@ -31,88 +30,156 @@ func Init() Config { listenPort := cfg.GetEnv("LISTEN_PORT", "9171") os.Setenv("LISTEN_PORT", listenPort) ac := cfg.Init() - url := cfg.GetEnv("API_URL", "https://api.github.com") + + appConfig := Config{ + &ac, + nil, + nil, + nil, + nil, + "", + nil, + } + + err := appConfig.SetAPIURL(cfg.GetEnv("API_URL", "https://api.github.com")) + if err != nil { + log.Errorf("Error initialising Configuration. Unable to parse API URL. Error: %v", err) + } repos := os.Getenv("REPOS") + if repos != "" { + appConfig.SetRepositories(strings.Split(repos, ", ")) + } orgs := os.Getenv("ORGS") + if orgs != "" { + appConfig.SetOrganisations(strings.Split(repos, ", ")) + } users := os.Getenv("USERS") + if users != "" { + appConfig.SetUsers(strings.Split(users, ", ")) + } tokenEnv := os.Getenv("GITHUB_TOKEN") tokenFile := os.Getenv("GITHUB_TOKEN_FILE") - token, err := getAuth(tokenEnv, tokenFile) - scraped, err := getScrapeURLs(url, repos, orgs, users) - - if err != nil { - log.Errorf("Error initialising Configuration, Error: %v", err) - } - - appConfig := Config{ - &ac, - url, - repos, - orgs, - users, - tokenEnv, - tokenFile, - token, - scraped, + if tokenEnv != "" { + appConfig.SetAPIToken(tokenEnv) + } else if tokenFile != "" { + err = appConfig.SetAPITokenFromFile(tokenFile) + if err != nil { + log.Errorf("Error initialising Configuration, Error: %v", err) + } } return appConfig } +// Returns the base APIURL +func (c *Config) APIURL() *url.URL { + return c.apiUrl +} + +// Returns a list of all object URLs to scrape +func (c *Config) TargetURLs() []string { + return c.targetURLs +} + +// Returns the oauth2 token for usage in http.request +func (c *Config) APIToken() string { + return c.apiToken +} + +// Sets the base API URL returning an error if the supplied string is not a valid URL +func (c *Config) SetAPIURL(u string) error { + ur, err := url.Parse(u) + c.apiUrl = ur + return err +} + +// Overrides the entire list of repositories +func (c *Config) SetRepositories(repos []string) { + c.repositories = repos + c.setScrapeURLs() +} + +// Overrides the entire list of organisations +func (c *Config) SetOrganisations(orgs []string) { + c.organisations = orgs + c.setScrapeURLs() +} + +// Overrides the entire list of users +func (c *Config) SetUsers(users []string) { + c.users = users + c.setScrapeURLs() +} + +// SetAPIToken accepts a string oauth2 token for usage in http.request +func (c *Config) SetAPIToken(token string) { + c.apiToken = token +} + +// SetAPITokenFromFile accepts a file containing an oauth2 token for usage in http.request +func (c *Config) SetAPITokenFromFile(tokenFile string) error { + b, err := ioutil.ReadFile(tokenFile) + if err != nil { + return err + } + c.apiToken = strings.TrimSpace(string(b)) + return nil +} + // Init populates the Config struct based on environmental runtime configuration // All URL's are added to the TargetURL's string array -func getScrapeURLs(apiURL, repos, orgs, users string) ([]string, error) { +func (c *Config) setScrapeURLs() error { urls := []string{} - opts := "?&per_page=100" // Used to set the Github API to return 100 results per page (max) + opts := map[string]string{"per_page": "100"} // Used to set the Github API to return 100 results per page (max) - if len(repos) == 0 && len(orgs) == 0 && len(users) == 0 { + if len(c.repositories) == 0 && len(c.organisations) == 0 && len(c.users) == 0 { log.Info("No targets specified. Only rate limit endpoint will be scraped") } // Append repositories to the array - if repos != "" { - rs := strings.Split(repos, ", ") - for _, x := range rs { - y := fmt.Sprintf("%s/repos/%s%s", apiURL, x, opts) - urls = append(urls, y) + if len(c.repositories) > 0 { + for _, x := range c.repositories { + y := *c.apiUrl + y.Path = path.Join(y.Path, "repos", x) + q := y.Query() + for k, v := range opts { + q.Add(k, v) + } + y.RawQuery = q.Encode() + urls = append(urls, y.String()) } } // Append github orginisations to the array - if orgs != "" { - o := strings.Split(orgs, ", ") - for _, x := range o { - y := fmt.Sprintf("%s/orgs/%s/repos%s", apiURL, x, opts) - urls = append(urls, y) + if len(c.organisations) > 0 { + for _, x := range c.organisations { + y := *c.apiUrl + y.Path = path.Join(y.Path, "orgs", x, "repos") + q := y.Query() + for k, v := range opts { + q.Add(k, v) + } + y.RawQuery = q.Encode() + urls = append(urls, y.String()) } } - if users != "" { - us := strings.Split(users, ", ") - for _, x := range us { - y := fmt.Sprintf("%s/users/%s/repos%s", apiURL, x, opts) - urls = append(urls, y) + if len(c.users) > 0 { + for _, x := range c.users { + y := *c.apiUrl + y.Path = path.Join(y.Path, "users", x, "repos") + q := y.Query() + for k, v := range opts { + q.Add(k, v) + } + y.RawQuery = q.Encode() + urls = append(urls, y.String()) } } - return urls, nil -} - -// getAuth returns oauth2 token as string for usage in http.request -func getAuth(token string, tokenFile string) (string, error) { - - if token != "" { - return token, nil - } else if tokenFile != "" { - b, err := ioutil.ReadFile(tokenFile) - if err != nil { - return "", err - } - return strings.TrimSpace(string(b)), err - - } + c.targetURLs = urls - return "", nil + return nil } diff --git a/exporter/gather.go b/exporter/gather.go index f149b777..7a72db2d 100644 --- a/exporter/gather.go +++ b/exporter/gather.go @@ -3,6 +3,7 @@ package exporter import ( "encoding/json" "fmt" + "path" "strconv" "strings" @@ -14,7 +15,7 @@ func (e *Exporter) gatherData() ([]*Datum, error) { data := []*Datum{} - responses, err := asyncHTTPGets(e.TargetURLs, e.APIToken) + responses, err := asyncHTTPGets(e.TargetURLs(), e.APIToken()) if err != nil { return data, err @@ -54,11 +55,10 @@ func (e *Exporter) gatherData() ([]*Datum, error) { // getRates obtains the rate limit data for requests against the github API. // Especially useful when operating without oauth and the subsequent lower cap. func (e *Exporter) getRates() (*RateLimits, error) { + u := *e.APIURL() + u.Path = path.Join(u.Path, "rate_limit") - rateEndPoint := ("/rate_limit") - url := fmt.Sprintf("%s%s", e.APIURL, rateEndPoint) - - resp, err := getHTTPResponse(url, e.APIToken) + resp, err := getHTTPResponse(u.String(), e.APIToken()) if err != nil { return &RateLimits{}, err } @@ -99,7 +99,7 @@ func getReleases(e *Exporter, url string, data *[]Release) { i := strings.Index(url, "?") baseURL := url[:i] releasesURL := baseURL + "/releases" - releasesResponse, err := asyncHTTPGets([]string{releasesURL}, e.APIToken) + releasesResponse, err := asyncHTTPGets([]string{releasesURL}, e.APIToken()) if err != nil { log.Errorf("Unable to obtain releases from API, Error: %s", err) @@ -112,7 +112,7 @@ func getPRs(e *Exporter, url string, data *[]Pull) { i := strings.Index(url, "?") baseURL := url[:i] pullsURL := baseURL + "/pulls" - pullsResponse, err := asyncHTTPGets([]string{pullsURL}, e.APIToken) + pullsResponse, err := asyncHTTPGets([]string{pullsURL}, e.APIToken()) if err != nil { log.Errorf("Unable to obtain pull requests from API, Error: %s", err) diff --git a/exporter/prometheus.go b/exporter/prometheus.go index 7f5c28f6..280c4f70 100644 --- a/exporter/prometheus.go +++ b/exporter/prometheus.go @@ -21,7 +21,7 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) { data := []*Datum{} var err error // Scrape the Data from Github - if len(e.TargetURLs) > 0 { + if len(e.TargetURLs()) > 0 { data, err = e.gatherData() if err != nil { log.Errorf("Error gathering Data from remote API: %v", err)