From 18b827e671f85aeab38ccd16e9c044a249d74327 Mon Sep 17 00:00:00 2001 From: Roman Date: Thu, 9 Dec 2021 12:50:04 +0300 Subject: [PATCH 1/4] feat(http): change method add headers to set headers --- pkg/drivers/http/driver.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/drivers/http/driver.go b/pkg/drivers/http/driver.go index e9fd343d6..158d37bcc 100644 --- a/pkg/drivers/http/driver.go +++ b/pkg/drivers/http/driver.go @@ -3,13 +3,14 @@ package http import ( "bytes" "context" - "github.com/MontFerret/ferret/pkg/runtime/logging" - "github.com/MontFerret/ferret/pkg/runtime/values" - "github.com/gobwas/glob" "io" "net/http" "net/url" + "github.com/MontFerret/ferret/pkg/runtime/logging" + "github.com/MontFerret/ferret/pkg/runtime/values" + "github.com/gobwas/glob" + "golang.org/x/net/html/charset" "github.com/PuerkitoBio/goquery" @@ -219,7 +220,7 @@ func (drv *Driver) makeRequest(ctx context.Context, req *http.Request, params dr params.Headers.ForEach(func(value []string, key string) bool { v := params.Headers.Get(key) - req.Header.Add(key, v) + req.Header.Set(key, v) logger. Debug(). From de2c660d69aae361298838dc4fffbdb47508e280 Mon Sep 17 00:00:00 2001 From: Roman Date: Fri, 10 Dec 2021 01:29:45 +0300 Subject: [PATCH 2/4] feat(options): add default timeout adn user options --- pkg/drivers/http/options.go | 9 +++++++++ pkg/drivers/http/options_test.go | 3 +++ 2 files changed, 12 insertions(+) diff --git a/pkg/drivers/http/options.go b/pkg/drivers/http/options.go index c7f084f69..156fab6a0 100644 --- a/pkg/drivers/http/options.go +++ b/pkg/drivers/http/options.go @@ -2,6 +2,7 @@ package http import ( stdhttp "net/http" + "time" "github.com/gobwas/glob" "github.com/sethgrid/pester" @@ -12,6 +13,7 @@ import ( var ( DefaultConcurrency = 1 DefaultMaxRetries = 5 + DefaultTimeout = time.Second * 30 ) type ( @@ -29,6 +31,7 @@ type ( Concurrency int HTTPCodesFilter []compiledStatusCodeFilter HTTPTransport *stdhttp.Transport + Timeout time.Duration } ) @@ -143,3 +146,9 @@ func WithCustomTransport(transport *stdhttp.Transport) Option { opts.HTTPTransport = transport } } + +func WithTimeout(duration time.Duration) Option { + return func(opts *Options) { + opts.Timeout = duration + } +} diff --git a/pkg/drivers/http/options_test.go b/pkg/drivers/http/options_test.go index 49902931a..505774745 100644 --- a/pkg/drivers/http/options_test.go +++ b/pkg/drivers/http/options_test.go @@ -30,6 +30,7 @@ func TestNewOptions(t *testing.T) { expectedMaxRetries := 2 expectedConcurrency := 10 expectedTransport := &stdhttp.Transport{} + expectedTimeout := time.Second * 5 opts := http.NewOptions([]http.Option{ http.WithCustomName(expectedName), @@ -69,6 +70,7 @@ func TestNewOptions(t *testing.T) { http.WithAllowedHTTPCode(401), http.WithAllowedHTTPCodes([]int{403, 404}), http.WithCustomTransport(expectedTransport), + http.WithTimeout(time.Second * 5), }) So(opts.Options, ShouldNotBeNil) So(opts.Name, ShouldEqual, expectedName) @@ -81,5 +83,6 @@ func TestNewOptions(t *testing.T) { So(opts.Concurrency, ShouldEqual, expectedConcurrency) So(opts.HTTPCodesFilter, ShouldHaveLength, 3) So(opts.HTTPTransport, ShouldEqual, expectedTransport) + So(opts.Timeout, ShouldEqual, expectedTimeout) }) } From 7380e897ebe3338e58609214198b0d41071d1762 Mon Sep 17 00:00:00 2001 From: Roman Date: Fri, 10 Dec 2021 01:30:52 +0300 Subject: [PATCH 3/4] feat(options): set default timeout --- pkg/drivers/http/options.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/drivers/http/options.go b/pkg/drivers/http/options.go index 156fab6a0..5203521c9 100644 --- a/pkg/drivers/http/options.go +++ b/pkg/drivers/http/options.go @@ -42,6 +42,7 @@ func NewOptions(setters []Option) *Options { opts.Backoff = pester.ExponentialBackoff opts.Concurrency = DefaultConcurrency opts.MaxRetries = DefaultMaxRetries + opts.Timeout = DefaultTimeout opts.HTTPCodesFilter = make([]compiledStatusCodeFilter, 0, 5) for _, setter := range setters { From 81220a7a853794f5f73db4101c3014e4aee6d16c Mon Sep 17 00:00:00 2001 From: Roman Date: Fri, 10 Dec 2021 01:31:23 +0300 Subject: [PATCH 4/4] feat(http): change init options adn remove trash code --- pkg/drivers/http/driver.go | 10 +++++----- pkg/drivers/http/driver_test.go | 27 --------------------------- 2 files changed, 5 insertions(+), 32 deletions(-) diff --git a/pkg/drivers/http/driver.go b/pkg/drivers/http/driver.go index 158d37bcc..805731d86 100644 --- a/pkg/drivers/http/driver.go +++ b/pkg/drivers/http/driver.go @@ -33,9 +33,6 @@ func NewDriver(opts ...Option) *Driver { drv.options = NewOptions(opts) drv.client = newHTTPClient(drv.options) - drv.client.Concurrency = drv.options.Concurrency - drv.client.MaxRetries = drv.options.MaxRetries - drv.client.Backoff = drv.options.Backoff return drv } @@ -43,6 +40,11 @@ func NewDriver(opts ...Option) *Driver { func newHTTPClient(options *Options) (httpClient *pester.Client) { httpClient = pester.New() + httpClient.Concurrency = options.Concurrency + httpClient.MaxRetries = options.MaxRetries + httpClient.Backoff = options.Backoff + httpClient.Timeout = options.Timeout + if options.HTTPTransport != nil { httpClient.Transport = options.HTTPTransport } @@ -55,8 +57,6 @@ func newHTTPClient(options *Options) (httpClient *pester.Client) { return } - httpClient = pester.NewExtendedClient(&http.Client{Transport: httpClient.Transport}) - return } diff --git a/pkg/drivers/http/driver_test.go b/pkg/drivers/http/driver_test.go index ce546f0a5..f6ca28ae1 100644 --- a/pkg/drivers/http/driver_test.go +++ b/pkg/drivers/http/driver_test.go @@ -39,15 +39,6 @@ func Test_newHTTPClientWithTransport(t *testing.T) { HTTPTransport: httpTransport, }}, }, - { - name: "check transport exist with pester.NewExtendedClient()", - args: args{options: &Options{ - Options: &drivers.Options{ - Proxy: "http://0.0.0.0", - }, - HTTPTransport: httpTransport, - }}, - }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -95,24 +86,6 @@ func Test_newHTTPClient(t *testing.T) { So(hc, ShouldBeNil) }) - - Convey("pester.NewExtend()", t, func() { - var ( - client = newHTTPClient(&Options{ - Options: &drivers.Options{ - Proxy: "http://0.0.0.0", - }, - }) - - rValue = reflect.ValueOf(client).Elem() - rField = rValue.Field(0) - ) - - rField = reflect.NewAt(rField.Type(), unsafe.Pointer(rField.UnsafeAddr())).Elem() - hc := rField.Interface().(*http.Client) - - So(hc, ShouldNotBeNil) - }) } func TestDriver_convertToUTF8(t *testing.T) {