Add more config parameters to HTTPClientSettings to be used by multip…

…le http-based exporters (#4188) * Added `MaxIdleConns`, `MaxIdleConnsPerHost` and `IdleConnTimeout` in HTTPClientSettings * Removed unnecessary validatioins form confighttp * Added MaxConnsPerHost parameter in confighttp * Changed MaxIdleConns and IdleConnTimeout parameter type to pointer variable * Modified the test case * Updated confighttp * Fixed linting * Make all new parameter type to pointer * Add DefaultHTTPClientSettings function in confighttp
open-telemetry · Dec 6, 2021 · a06ca26 · a06ca26
1 parent 83206c5
commit a06ca26
Show file tree

Hide file tree

Showing 3 changed files with 114 additions and 3 deletions.
diff --git a/config/confighttp/README.md b/config/confighttp/README.md
@@ -20,6 +20,10 @@ README](../configtls/README.md).
 - [`read_buffer_size`](https://golang.org/pkg/net/http/#Transport)
 - [`timeout`](https://golang.org/pkg/net/http/#Client)
 - [`write_buffer_size`](https://golang.org/pkg/net/http/#Transport)
+- [`max_idle_conns`](https://golang.org/pkg/net/http/#Transport)
+- [`max_idle_conns_per_host`](https://golang.org/pkg/net/http/#Transport)
+- [`max_conns_per_host`](https://golang.org/pkg/net/http/#Transport)
+- [`idle_conn_timeout`](https://golang.org/pkg/net/http/#Transport)
 
 Example:
 

diff --git a/config/confighttp/confighttp.go b/config/confighttp/confighttp.go
@@ -58,6 +58,38 @@ type HTTPClientSettings struct {
 
 	// Auth configuration for outgoing HTTP calls.
 	Auth *configauth.Authentication `mapstructure:"auth,omitempty"`
+
+	// MaxIdleConns is used to set a limit to the maximum idle HTTP connections the client can keep open.
+	// There's an already set value, and we want to override it only if an explicit value provided
+	MaxIdleConns *int `mapstructure:"max_idle_conns"`
+
+	// MaxIdleConnsPerHost is used to set a limit to the maximum idle HTTP connections the host can keep open.
+	// There's an already set value, and we want to override it only if an explicit value provided
+	MaxIdleConnsPerHost *int `mapstructure:"max_idle_conns_per_host"`
+
+	// MaxConnsPerHost limits the total number of connections per host, including connections in the dialing,
+	// active, and idle states.
+	// There's an already set value, and we want to override it only if an explicit value provided
+	MaxConnsPerHost *int `mapstructure:"max_conns_per_host"`
+
+	// IdleConnTimeout is the maximum amount of time a connection will remain open before closing itself.
+	// There's an already set value, and we want to override it only if an explicit value provided
+	IdleConnTimeout *time.Duration `mapstructure:"idle_conn_timeout"`
+}
+
+// DefaultHTTPClientSettings returns HTTPClientSettings type object with
+// the default values of 'MaxIdleConns' and 'IdleConnTimeout'.
+// Other config options are not added as they are initialized with 'zero value' by GoLang as default.
+// We encourage to use this function to create an object of HTTPClientSettings.
+func DefaultHTTPClientSettings() HTTPClientSettings {
+	// The default values are taken from the values of 'DefaultTransport' of 'http' package.
+	maxIdleConns := 100
+	idleConnTimeout := 90 * time.Second
+
+	return HTTPClientSettings{
+		MaxIdleConns:    &maxIdleConns,
+		IdleConnTimeout: &idleConnTimeout,
+	}
 }
 
 // ToClient creates an HTTP client.
@@ -77,6 +109,22 @@ func (hcs *HTTPClientSettings) ToClient(ext map[config.ComponentID]component.Ext
 		transport.WriteBufferSize = hcs.WriteBufferSize
 	}
 
+	if hcs.MaxIdleConns != nil {
+		transport.MaxIdleConns = *hcs.MaxIdleConns
+	}
+
+	if hcs.MaxIdleConnsPerHost != nil {
+		transport.MaxIdleConnsPerHost = *hcs.MaxIdleConnsPerHost
+	}
+
+	if hcs.MaxConnsPerHost != nil {
+		transport.MaxConnsPerHost = *hcs.MaxConnsPerHost
+	}
+
+	if hcs.IdleConnTimeout != nil {
+		transport.IdleConnTimeout = *hcs.IdleConnTimeout
+	}
+
 	clientTransport := (http.RoundTripper)(transport)
 	if len(hcs.Headers) > 0 {
 		clientTransport = &headerRoundTripper{

diff --git a/config/confighttp/confighttp_test.go b/config/confighttp/confighttp_test.go
@@ -48,6 +48,10 @@ func TestAllHTTPClientSettings(t *testing.T) {
 	ext := map[config.ComponentID]component.Extension{
 		config.NewComponentID("testauth"): &configauth.MockClientAuthenticator{ResultRoundTripper: &customRoundTripper{}},
 	}
+	maxIdleConns := 50
+	maxIdleConnsPerHost := 40
+	maxConnsPerHost := 45
+	idleConnTimeout := 30 * time.Second
 	tests := []struct {
 		name        string
 		settings    HTTPClientSettings
@@ -60,9 +64,13 @@ func TestAllHTTPClientSettings(t *testing.T) {
 				TLSSetting: configtls.TLSClientSetting{
 					Insecure: false,
 				},
-				ReadBufferSize:     1024,
-				WriteBufferSize:    512,
-				CustomRoundTripper: func(next http.RoundTripper) (http.RoundTripper, error) { return next, nil },
+				ReadBufferSize:      1024,
+				WriteBufferSize:     512,
+				MaxIdleConns:        &maxIdleConns,
+				MaxIdleConnsPerHost: &maxIdleConnsPerHost,
+				MaxConnsPerHost:     &maxConnsPerHost,
+				IdleConnTimeout:     &idleConnTimeout,
+				CustomRoundTripper:  func(next http.RoundTripper) (http.RoundTripper, error) { return next, nil },
 			},
 			shouldError: false,
 		},
@@ -92,10 +100,61 @@ func TestAllHTTPClientSettings(t *testing.T) {
 			transport := client.Transport.(*http.Transport)
 			assert.EqualValues(t, 1024, transport.ReadBufferSize)
 			assert.EqualValues(t, 512, transport.WriteBufferSize)
+			assert.EqualValues(t, 50, transport.MaxIdleConns)
+			assert.EqualValues(t, 40, transport.MaxIdleConnsPerHost)
+			assert.EqualValues(t, 45, transport.MaxConnsPerHost)
+			assert.EqualValues(t, 30*time.Second, transport.IdleConnTimeout)
+
+		})
+	}
+}
+
+func TestPartialHTTPClientSettings(t *testing.T) {
+	ext := map[config.ComponentID]component.Extension{
+		config.NewComponentID("testauth"): &configauth.MockClientAuthenticator{ResultRoundTripper: &customRoundTripper{}},
+	}
+	tests := []struct {
+		name        string
+		settings    HTTPClientSettings
+		shouldError bool
+	}{
+		{
+			name: "valid_partial_settings",
+			settings: HTTPClientSettings{
+				Endpoint: "localhost:1234",
+				TLSSetting: configtls.TLSClientSetting{
+					Insecure: false,
+				},
+				ReadBufferSize:     1024,
+				WriteBufferSize:    512,
+				CustomRoundTripper: func(next http.RoundTripper) (http.RoundTripper, error) { return next, nil },
+			},
+			shouldError: false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			client, err := test.settings.ToClient(ext)
+			assert.NoError(t, err)
+			transport := client.Transport.(*http.Transport)
+			assert.EqualValues(t, 1024, transport.ReadBufferSize)
+			assert.EqualValues(t, 512, transport.WriteBufferSize)
+			assert.EqualValues(t, 100, transport.MaxIdleConns)
+			assert.EqualValues(t, 0, transport.MaxIdleConnsPerHost)
+			assert.EqualValues(t, 0, transport.MaxConnsPerHost)
+			assert.EqualValues(t, 90*time.Second, transport.IdleConnTimeout)
+
 		})
 	}
 }
 
+func TestDefaultHTTPClientSettings(t *testing.T) {
+	httpClientSettings := DefaultHTTPClientSettings()
+	assert.EqualValues(t, 100, *httpClientSettings.MaxIdleConns)
+	assert.EqualValues(t, 90*time.Second, *httpClientSettings.IdleConnTimeout)
+}
+
 func TestHTTPClientSettingsError(t *testing.T) {
 	tests := []struct {
 		settings HTTPClientSettings