From 6c3b6a12842d543031a673258fb5bd777c2a331e Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 9 May 2022 19:49:22 +0200 Subject: [PATCH 1/2] Adding SNI override via request annotations --- integration_tests/http/get-override-sni.yaml | 18 ++++++++++ v2/cmd/integration-test/http.go | 23 ++++++++++++ v2/go.mod | 2 +- v2/go.sum | 2 ++ .../protocols/headless/engine/http_client.go | 1 + v2/pkg/protocols/http/build_request.go | 5 ++- .../http/httpclientpool/clientpool.go | 1 + v2/pkg/protocols/http/request_annotations.go | 35 +++++++++++++++++-- .../reporting/exporters/es/elasticsearch.go | 1 + 9 files changed, 83 insertions(+), 5 deletions(-) create mode 100644 integration_tests/http/get-override-sni.yaml diff --git a/integration_tests/http/get-override-sni.yaml b/integration_tests/http/get-override-sni.yaml new file mode 100644 index 0000000000..61239ee5ce --- /dev/null +++ b/integration_tests/http/get-override-sni.yaml @@ -0,0 +1,18 @@ +id: basic-raw-http-example + +info: + name: Test RAW GET Template + author: pdteam + severity: info + +requests: + - raw: + - | + @tls-sni:request.host + GET / HTTP/1.1 + Host: test + + matchers: + - type: word + words: + - "test-ok" \ No newline at end of file diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index c13c497d08..b4407968c2 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -48,6 +48,7 @@ var httpTestcases = map[string]testutils.TestCase{ "http/stop-at-first-match.yaml": &httpStopAtFirstMatch{}, "http/stop-at-first-match-with-extractors.yaml": &httpStopAtFirstMatchWithExtractors{}, "http/variables.yaml": &httpVariables{}, + "http/get-override-sni.yaml": &httpSniAnnotation{}, } type httpInteractshRequest struct{} @@ -810,3 +811,25 @@ func (h *httpVariables) Execute(filePath string) error { return expectResultsCount(results, 1) } + +type httpSniAnnotation struct{} + +// Execute executes a test case and returns an error if occurred +func (h *httpSniAnnotation) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + if r.TLS.ServerName == "test" { + _, _ = w.Write([]byte("test-ok")) + } else { + _, _ = w.Write([]byte("test-ko")) + } + }) + ts := httptest.NewTLSServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) + if err != nil { + return err + } + return expectResultsCount(results, 1) +} diff --git a/v2/go.mod b/v2/go.mod index 3048b706be..66be023dfa 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -27,7 +27,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/cryptoutil v1.0.0 - github.com/projectdiscovery/fastdialer v0.0.15-0.20220127193345-f06b0fd54d47 + github.com/projectdiscovery/fastdialer v0.0.16-0.20220509174423-0e57a7c8cf83 github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 github.com/projectdiscovery/fileutil v0.0.0-20220427234316-40b2541a84b8 github.com/projectdiscovery/goflags v0.0.8-0.20220412061559-5119d6086323 diff --git a/v2/go.sum b/v2/go.sum index 607340d452..bc4eb71d4a 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -436,6 +436,8 @@ github.com/projectdiscovery/cryptoutil v1.0.0/go.mod h1:VJvSNE8f8A1MgpjgAL2GPJSQ github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0= github.com/projectdiscovery/fastdialer v0.0.15-0.20220127193345-f06b0fd54d47 h1:TUsZiwez9uFmph1hlTsiH7rdB+wi4524+lMuV2z6FaM= github.com/projectdiscovery/fastdialer v0.0.15-0.20220127193345-f06b0fd54d47/go.mod h1:GbQvP1ezGlQn0af3lVcl08b5eRQu960T7A9pwazybSo= +github.com/projectdiscovery/fastdialer v0.0.16-0.20220509174423-0e57a7c8cf83 h1:1hzvl0lsWpvQ8nn1s9YMyBjO13/Z+f/T4W2jroOohfo= +github.com/projectdiscovery/fastdialer v0.0.16-0.20220509174423-0e57a7c8cf83/go.mod h1:wn6jSJ1fIO6kLplFEbFIkRB6Kj/Q6VngnzKuBHLVPiI= github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 h1:NwD1R/du1dqrRKN3SJl9kT6tN3K9puuWFXEvYF2ihew= github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08/go.mod h1:paLCnwV8sL7ppqIwVQodQrk3F6mnWafwTDwRd7ywZwQ= github.com/projectdiscovery/fileutil v0.0.0-20210914153648-31f843feaad4/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go index 9b1c5b0df5..adff97682a 100644 --- a/v2/pkg/protocols/headless/engine/http_client.go +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -37,6 +37,7 @@ func newHttpClient(options *types.Options) (*http.Client, error) { transport := &http.Transport{ DialContext: dialer.Dial, + DialTLSContext: dialer.DialTLS, MaxIdleConns: 500, MaxIdleConnsPerHost: 500, MaxConnsPerHost: 500, diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 2fcc062cb3..b338d8a076 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -288,7 +288,10 @@ func (r *requestGenerator) handleRawWithPayloads(ctx context.Context, rawRequest return nil, err } - parseAnnotations(rawRequest, req) + if reqWithAnnotations, hasAnnotations := parseAnnotations(rawRequest, req); hasAnnotations { + req = reqWithAnnotations + request = request.WithContext(req.Context()) + } return &generatedRequest{request: request, meta: generatorValues, original: r.request, dynamicValues: finalValues, interactshURLs: r.interactshURLs}, nil } diff --git a/v2/pkg/protocols/http/httpclientpool/clientpool.go b/v2/pkg/protocols/http/httpclientpool/clientpool.go index 81f25885f7..f14a5f500f 100644 --- a/v2/pkg/protocols/http/httpclientpool/clientpool.go +++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go @@ -187,6 +187,7 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl transport := &http.Transport{ DialContext: Dialer.Dial, + DialTLSContext: Dialer.DialTLS, MaxIdleConns: maxIdleConns, MaxIdleConnsPerHost: maxIdleConnsPerHost, MaxConnsPerHost: maxConnsPerHost, diff --git a/v2/pkg/protocols/http/request_annotations.go b/v2/pkg/protocols/http/request_annotations.go index 34464fd90b..8228fec44e 100644 --- a/v2/pkg/protocols/http/request_annotations.go +++ b/v2/pkg/protocols/http/request_annotations.go @@ -1,22 +1,33 @@ package http import ( + "context" "net" "net/http" "regexp" "strings" + "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/iputil" "github.com/projectdiscovery/stringsutil" "github.com/projectdiscovery/urlutil" ) -// @Host:target overrides the input target with the annotated one (similar to self-contained requests) -var reHostAnnotation = regexp.MustCompile(`(?m)^@Host:\s*(.+)\s*$`) +var ( + // @Host:target overrides the input target with the annotated one (similar to self-contained requests) + reHostAnnotation = regexp.MustCompile(`(?m)^@Host:\s*(.+)\s*$`) + // @tls-sni:target overrides the input target with the annotated one + // special values: + // request.host: takes the value from the host header + // target: overiddes with the specific value + reSniAnnotation = regexp.MustCompile(`(?m)^@tls-sni:\s*(.+)\s*$`) +) // parseAnnotations and override requests settings -func parseAnnotations(rawRequest string, request *http.Request) { +func parseAnnotations(rawRequest string, request *http.Request) (*http.Request, bool) { // parse request for known ovverride annotations + var modified bool + // @Host:target if hosts := reHostAnnotation.FindStringSubmatch(rawRequest); len(hosts) > 0 { value := strings.TrimSpace(hosts[1]) // handle scheme @@ -39,7 +50,25 @@ func parseAnnotations(rawRequest string, request *http.Request) { } request.URL.Host = hostPort } + modified = true + } + + // @tls-sni:target + if hosts := reSniAnnotation.FindStringSubmatch(rawRequest); len(hosts) > 0 { + value := strings.TrimSpace(hosts[1]) + value = stringsutil.TrimPrefixAny(value, "http://", "https://") + if idxForwardSlash := strings.Index(value, "/"); idxForwardSlash >= 0 { + value = value[:idxForwardSlash] + } + + if stringsutil.EqualFoldAny(value, "request.host") { + value = request.Host + } + ctx := context.WithValue(request.Context(), fastdialer.SniName, value) + request = request.Clone(ctx) + modified = true } + return request, modified } func isHostPort(value string) bool { diff --git a/v2/pkg/reporting/exporters/es/elasticsearch.go b/v2/pkg/reporting/exporters/es/elasticsearch.go index 96adac1b2b..545ab9487c 100644 --- a/v2/pkg/reporting/exporters/es/elasticsearch.go +++ b/v2/pkg/reporting/exporters/es/elasticsearch.go @@ -63,6 +63,7 @@ func New(option *Options) (*Exporter, error) { MaxIdleConns: 10, MaxIdleConnsPerHost: 10, DialContext: protocolstate.Dialer.Dial, + DialTLSContext: protocolstate.Dialer.DialTLS, TLSClientConfig: &tls.Config{InsecureSkipVerify: option.SSLVerification}, }, } From 32c8e4eaf88f777935cab91cd1399bc3203d582c Mon Sep 17 00:00:00 2001 From: mzack Date: Thu, 12 May 2022 11:04:54 +0200 Subject: [PATCH 2/2] adding cli flag priority --- v2/pkg/protocols/common/protocolstate/state.go | 1 + 1 file changed, 1 insertion(+) diff --git a/v2/pkg/protocols/common/protocolstate/state.go b/v2/pkg/protocols/common/protocolstate/state.go index a2fc4db8cb..604ae732fd 100644 --- a/v2/pkg/protocols/common/protocolstate/state.go +++ b/v2/pkg/protocols/common/protocolstate/state.go @@ -21,6 +21,7 @@ func Init(options *types.Options) error { } opts.WithDialerHistory = true opts.WithZTLS = options.ZTLS + opts.SNIName = options.SNI dialer, err := fastdialer.NewDialer(opts) if err != nil { return errors.Wrap(err, "could not create dialer")