Bugfix/#638 http driver multiple requsets

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/corpix/uarand v0.1.1
 	github.com/gobwas/glob v0.2.3
 	github.com/gorilla/css v1.0.0
+	github.com/jarcoal/httpmock v1.0.8
 	github.com/mafredri/cdp v0.32.0
 	github.com/pkg/errors v0.9.1
 	github.com/rs/zerolog v1.23.0

go.sum

@@ -35,6 +35,8 @@ github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
 github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/jarcoal/httpmock v1.0.8 h1:8kI16SoO6LQKgPE7PvQuV+YuD/inwHd7fOOe2zMbo4k=
+github.com/jarcoal/httpmock v1.0.8/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
 github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=

pkg/drivers/http/driver_test.go

@@ -2,18 +2,20 @@ package http
 
 import (
 	"bytes"
+	"context"
 	"crypto/tls"
-	"github.com/MontFerret/ferret/pkg/drivers"
 	"io"
 	"io/ioutil"
 	"net/http"
 	"reflect"
 	"testing"
 	"unsafe"
 
+	"github.com/jarcoal/httpmock"
+	. "github.com/smartystreets/goconvey/convey"
 	"golang.org/x/text/encoding/charmap"
 
-	"github.com/smartystreets/goconvey/convey"
+	"github.com/MontFerret/ferret/pkg/drivers"
 )
 
 func Test_newHTTPClientWithTransport(t *testing.T) {
@@ -49,7 +51,7 @@ func Test_newHTTPClientWithTransport(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			convey.Convey(tt.name, t, func() {
+			Convey(tt.name, t, func() {
 				var (
 					transport *http.Transport
 					client    = newHTTPClient(tt.args.options)
@@ -68,15 +70,15 @@ func Test_newHTTPClientWithTransport(t *testing.T) {
 
 				verify := transport.TLSClientConfig.InsecureSkipVerify
 
-				convey.So(verify, convey.ShouldBeTrue)
+				So(verify, ShouldBeTrue)
 			})
 		})
 	}
 }
 
 func Test_newHTTPClient(t *testing.T) {
 
-	convey.Convey("pester.New()", t, func() {
+	Convey("pester.New()", t, func() {
 		var (
 			client = newHTTPClient(&Options{
 				Options: &drivers.Options{
@@ -91,10 +93,10 @@ func Test_newHTTPClient(t *testing.T) {
 		rField = reflect.NewAt(rField.Type(), unsafe.Pointer(rField.UnsafeAddr())).Elem()
 		hc := rField.Interface().(*http.Client)
 
-		convey.So(hc, convey.ShouldBeNil)
+		So(hc, ShouldBeNil)
 	})
 
-	convey.Convey("pester.NewExtend()", t, func() {
+	Convey("pester.NewExtend()", t, func() {
 		var (
 			client = newHTTPClient(&Options{
 				Options: &drivers.Options{
@@ -109,7 +111,7 @@ func Test_newHTTPClient(t *testing.T) {
 		rField = reflect.NewAt(rField.Type(), unsafe.Pointer(rField.UnsafeAddr())).Elem()
 		hc := rField.Interface().(*http.Client)
 
-		convey.So(hc, convey.ShouldNotBeNil)
+		So(hc, ShouldNotBeNil)
 	})
 }
 
@@ -139,7 +141,7 @@ func TestDriver_convertToUTF8(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			drv := &Driver{}
 
-			convey.Convey(tt.name, t, func() {
+			Convey(tt.name, t, func() {
 
 				data, err := ioutil.ReadAll(bytes.NewBufferString(tt.args.inputData))
 				if err != nil {
@@ -157,15 +159,35 @@ func TestDriver_convertToUTF8(t *testing.T) {
 				encodedData = encodedData[:nDst]
 
 				gotData, err := drv.convertToUTF8(bytes.NewReader(encodedData), tt.args.srcCharset)
-				convey.So(err, convey.ShouldBeNil)
+				So(err, ShouldBeNil)
 
 				outData, err := ioutil.ReadAll(gotData)
-				convey.So(err, convey.ShouldBeNil)
+				So(err, ShouldBeNil)
 
-				convey.So(string(outData), convey.ShouldEqual, tt.expected)
+				So(string(outData), ShouldEqual, tt.expected)
 
 			})
 
 		})
 	}
 }
+
+func TestDriver_Concurrency(t *testing.T) {
+	Convey("Should make only 1 request", t, func() {
+		httpmock.Activate()
+		defer httpmock.DeactivateAndReset()
+
+		httpmock.RegisterResponder("GET", "http://localhost:1111",
+			httpmock.NewStringResponder(200, `<!DOCTYPE html><html><head></head><body></body></html>`))
+
+		drv := NewDriver()
+
+		page, err := drv.Open(context.Background(), drivers.Params{
+			URL: "http://localhost:1111",
+		})
+
+		So(err, ShouldBeNil)
+		So(page, ShouldNotBeNil)
+		So(httpmock.GetTotalCallCount(), ShouldEqual, 1)
+	})
+}

pkg/drivers/http/options.go

@@ -10,7 +10,7 @@ import (
 )
 
 var (
-	DefaultConcurrency = 3
+	DefaultConcurrency = 1
 	DefaultMaxRetries  = 5
 )