From fbb61a0e8132383921183d9de8f8280096e791b6 Mon Sep 17 00:00:00 2001 From: Quest Henkart Date: Sat, 9 Oct 2021 17:09:24 +0200 Subject: [PATCH 01/14] primary code transfer and expansion --- helpers/inbound/inbound.go | 116 ++++++++++++++++++++++++++++++++----- 1 file changed, 103 insertions(+), 13 deletions(-) diff --git a/helpers/inbound/inbound.go b/helpers/inbound/inbound.go index cd95e242..796b0e64 100644 --- a/helpers/inbound/inbound.go +++ b/helpers/inbound/inbound.go @@ -1,6 +1,7 @@ package inbound import ( + "encoding/json" "io" "io/ioutil" "mime" @@ -9,20 +10,54 @@ import ( "strings" ) +// ParsedEmail defines a multipart parsed email +// Body and Attachments are only populated if the Raw option is checked on the SendGrid inbound configuration and are named for backwards compatability type ParsedEmail struct { - Headers map[string]string - Body map[string]string + // Please see https://docs.sendgrid.com/for-developers/parsing-email/setting-up-the-inbound-parse-webhook to see the available fields in the email headers + // all fields listed there are available within the headers map except for text which lives in the TextBody field + Headers map[string]string + // Primary email body parsed with \n. A common approach is to Split by the \n to bring every line of the email into a string array + TextBody string + ParsedAttachments map[string]*EmailAttachment + + // Raw only Attachments map[string][]byte - rawRequest *http.Request + Body map[string]string + + rawRequest *http.Request + withAttachments bool +} + +// EmailAttachment defines information related to an email attachment +type EmailAttachment struct { + File multipart.File `json:"-"` + Filename string `json:"filename"` + Size int64 `json:"-"` + ContentType string `json:"type"` } +// Parse parses an email using Go's multipart parser and populates the headers, body func Parse(request *http.Request) (*ParsedEmail, error) { result := ParsedEmail{ - Headers: make(map[string]string), - Body: make(map[string]string), - Attachments: make(map[string][]byte), - rawRequest: request, + Headers: make(map[string]string), + ParsedAttachments: make(map[string]*EmailAttachment), + rawRequest: request, + withAttachments: false, + } + + err := result.parse() + return &result, err +} + +// ParseWithAttachments parses an email using Go's multipart parser and populates the headers, body and processes attachments +func ParseWithAttachments(request *http.Request) (*ParsedEmail, error) { + result := ParsedEmail{ + Headers: make(map[string]string), + ParsedAttachments: make(map[string]*EmailAttachment), + rawRequest: request, + withAttachments: true, } + err := result.parse() return &result, err } @@ -32,13 +67,68 @@ func (email *ParsedEmail) parse() error { if err != nil { return err } - emails := email.rawRequest.MultipartForm.Value["email"] - headers := email.rawRequest.MultipartForm.Value["headers"] - if len(headers) > 0 { - email.parseHeaders(headers[0]) + + values := email.rawRequest.MultipartForm.Value + + // parse included headers + if len(values["headers"]) > 0 { + email.parseHeaders(values["headers"][0]) + } + + // apply the rest of the SendGrid fields to the headers map + for k, v := range values { + if k == "text" || k == "email" { + continue + } + + if len(v) > 0 { + email.Headers[k] = v[0] + } + } + + // apply the plain text body + if len(values["text"]) > 0 { + email.TextBody = values["text"][0] + } + + // only included if the raw box is checked + if len(values["email"]) > 0 { + email.parseRawEmail(values["email"][0]) + } + + // if the client chose not to parse attachments, return as is + if !email.withAttachments { + return nil + } + + return email.parseAttachments(values) +} + +func (email *ParsedEmail) parseAttachments(values map[string][]string) error { + if len(values["attachment-info"]) != 1 { + return nil + } + // unmarshal the sendgrid parsed aspects of the email attachment into the attachment struct + if err := json.Unmarshal([]byte(values["attachment-info"][0]), &email.ParsedAttachments); err != nil { + return err } - if len(emails) > 0 { - email.parseRawEmail(emails[0]) + + // range through the multipart files + for key, val := range email.rawRequest.MultipartForm.File { + // open the attachment file for processing + file, err := val[0].Open() + if err != nil { + return err + } + + // add the actual file and the size to the parsed files + email.ParsedAttachments[key].File = file + email.ParsedAttachments[key].Size = val[0].Size + + // if the file does not have a name. give it Untitled + if email.ParsedAttachments[key].Filename == "" { + email.ParsedAttachments[key].Filename = "Untitled" + } } return nil From 99ecfd045aafc746be4ee9d0258149befcf9f7ee Mon Sep 17 00:00:00 2001 From: Quest Henkart Date: Sat, 9 Oct 2021 17:15:52 +0200 Subject: [PATCH 02/14] add validation method --- helpers/inbound/inbound.go | 41 +++++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/helpers/inbound/inbound.go b/helpers/inbound/inbound.go index 796b0e64..7cad8da4 100644 --- a/helpers/inbound/inbound.go +++ b/helpers/inbound/inbound.go @@ -2,6 +2,7 @@ package inbound import ( "encoding/json" + "fmt" "io" "io/ioutil" "mime" @@ -25,6 +26,7 @@ type ParsedEmail struct { Body map[string]string rawRequest *http.Request + rawValues map[string][]string withAttachments bool } @@ -68,15 +70,15 @@ func (email *ParsedEmail) parse() error { return err } - values := email.rawRequest.MultipartForm.Value + email.rawValues = email.rawRequest.MultipartForm.Value // parse included headers - if len(values["headers"]) > 0 { - email.parseHeaders(values["headers"][0]) + if len(email.rawValues["headers"]) > 0 { + email.parseHeaders(email.rawValues["headers"][0]) } // apply the rest of the SendGrid fields to the headers map - for k, v := range values { + for k, v := range email.rawValues { if k == "text" || k == "email" { continue } @@ -87,13 +89,13 @@ func (email *ParsedEmail) parse() error { } // apply the plain text body - if len(values["text"]) > 0 { - email.TextBody = values["text"][0] + if len(email.rawValues["text"]) > 0 { + email.TextBody = email.rawValues["text"][0] } // only included if the raw box is checked - if len(values["email"]) > 0 { - email.parseRawEmail(values["email"][0]) + if len(email.rawValues["email"]) > 0 { + email.parseRawEmail(email.rawValues["email"][0]) } // if the client chose not to parse attachments, return as is @@ -101,7 +103,7 @@ func (email *ParsedEmail) parse() error { return nil } - return email.parseAttachments(values) + return email.parseAttachments(email.rawValues) } func (email *ParsedEmail) parseAttachments(values map[string][]string) error { @@ -201,3 +203,24 @@ func (email *ParsedEmail) parseHeaders(headers string) { email.Headers[splitHeader[0]] = splitHeader[1] } } + +// Validate validates the DKIM and SPF scores to ensure that the email client and address was not spoofed +func (email *ParsedEmail) Validate() error { + if len(email.rawValues["dkim"]) == 0 || len(email.rawValues["SPF"]) == 0 { + return fmt.Errorf("missing DKIM and SPF score") + } + + for _, val := range email.rawValues["dkim"] { + if !strings.Contains(val, "pass") { + return fmt.Errorf("DKIM validation failed") + } + } + + for _, val := range email.rawValues["SPF"] { + if !strings.Contains(val, "pass") { + return fmt.Errorf("SPF validation failed") + } + } + + return nil +} From 1835899188d0a6fe65aa824eb521263168834f19 Mon Sep 17 00:00:00 2001 From: Quest Henkart Date: Sat, 9 Oct 2021 17:18:57 +0200 Subject: [PATCH 03/14] pass existing tests --- helpers/inbound/inbound.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/helpers/inbound/inbound.go b/helpers/inbound/inbound.go index 7cad8da4..80bda4ea 100644 --- a/helpers/inbound/inbound.go +++ b/helpers/inbound/inbound.go @@ -43,8 +43,12 @@ func Parse(request *http.Request) (*ParsedEmail, error) { result := ParsedEmail{ Headers: make(map[string]string), ParsedAttachments: make(map[string]*EmailAttachment), - rawRequest: request, - withAttachments: false, + + Body: make(map[string]string), + Attachments: make(map[string][]byte), + + rawRequest: request, + withAttachments: false, } err := result.parse() @@ -56,8 +60,11 @@ func ParseWithAttachments(request *http.Request) (*ParsedEmail, error) { result := ParsedEmail{ Headers: make(map[string]string), ParsedAttachments: make(map[string]*EmailAttachment), - rawRequest: request, - withAttachments: true, + + Body: make(map[string]string), + Attachments: make(map[string][]byte), + rawRequest: request, + withAttachments: true, } err := result.parse() @@ -179,6 +186,7 @@ func (email *ParsedEmail) parseRawEmail(rawEmail string) error { if err != nil { return err } + email.Body[header] = string(b) } } From 4bed0d0588393e2a3f561df9bcc994faf7f9b2d1 Mon Sep 17 00:00:00 2001 From: Quest Henkart Date: Sat, 9 Oct 2021 17:33:42 +0200 Subject: [PATCH 04/14] add a test for validation --- helpers/inbound/inbound_test.go | 54 ++++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/helpers/inbound/inbound_test.go b/helpers/inbound/inbound_test.go index b8b0019e..5b906409 100644 --- a/helpers/inbound/inbound_test.go +++ b/helpers/inbound/inbound_test.go @@ -60,12 +60,13 @@ func TestParse(t *testing.T) { email, err := Parse(req) if test.expectedError != nil { assert.Error(subTest, err, "expected an error to occur") - } else { - assert.NoError(subTest, err, "did NOT expect an error to occur") - - from := "Example User " - assert.Equalf(subTest, email.Headers["From"], from, "Expected From: %s, Got: %s", from, email.Headers["From"]) + return } + + assert.NoError(subTest, err, "did NOT expect an error to occur") + + from := "Example User " + assert.Equalf(subTest, email.Headers["From"], from, "Expected From: %s, Got: %s", from, email.Headers["From"]) }) } } @@ -129,3 +130,46 @@ Content-Transfer-Encoding: quoted-printable // Content-Type multipart/mixed; boundary=TwiLIo // Hello Twilio SendGrid! } + +func TestValidate(t *testing.T) { + tests := []struct { + name string + values map[string][]string + expectedError error + }{ + { + name: "MissingHeaders", + values: map[string][]string{}, + expectedError: fmt.Errorf("missing DKIM and SPF score"), + }, + { + name: "FailedDkim", + values: map[string][]string{"dkim": {"pass", "fail", "pass"}, "SPF": {"pass"}}, + expectedError: fmt.Errorf("DKIM validation failed"), + }, + { + name: "FailedSpf", + values: map[string][]string{"dkim": {"pass", "pass", "pass"}, "SPF": {"pass", "fail", "pass"}}, + expectedError: fmt.Errorf("SPF validation failed"), + }, + { + name: "success", + values: map[string][]string{"dkim": {"pass", "pass", "pass"}, "SPF": {"pass", "pass", "pass"}}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(subTest *testing.T) { + //Load POST body + email := ParsedEmail{rawValues: test.values} + err := email.Validate() + + if test.expectedError != nil { + assert.EqualError(subTest, test.expectedError, err.Error()) + return + } + + assert.NoError(subTest, err, "did NOT expect an error to occur") + }) + } +} From 0dd30bcb5790e4eb0d377c3f91bafa23fe4e22cf Mon Sep 17 00:00:00 2001 From: Quest Henkart Date: Sat, 9 Oct 2021 18:18:02 +0200 Subject: [PATCH 05/14] add parsed fields and update docs --- helpers/inbound/README.md | 38 +++++++++++++++++++++++++++++++++----- helpers/inbound/inbound.go | 18 +++++++++++++----- 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/helpers/inbound/README.md b/helpers/inbound/README.md index b7c06e86..683939e2 100644 --- a/helpers/inbound/README.md +++ b/helpers/inbound/README.md @@ -2,10 +2,25 @@ ## Table of Contents +* [Fields](#fields) * [Example Usage](#example-usage) * [Testing the Source Code](#testing) * [Contributing](#contributing) +# Fields +### parsedEmail.ParsedValues + Please see [Send Grid Docs](https://docs.sendgrid.com/for-developers/parsing-email/setting-up-the-inbound-parse-webhook) to see what fields are available and preparsed by SendGrid. Use these fields over the Headers as they are parsed by SendGrid and gauranteed to be consistent + +### parsedEmail.TextBody + this field will satisfy most cases. SendGrid pre-parses the body into a plain text string separated with \n + +### parsedEmail.Body and parsedEmail.Attachments + are populated *only* when the raw option is checked in the SendGrid Dashboard. However unless you need the raw HTML body, it is not necessary. The fields are named as they are for backward compatability + +### parsedEmail.Headers + these may change depending on the email client and are not pre-parsed by SendGrid, use carefully + + # Example Usage ```go @@ -20,27 +35,40 @@ import ( ) func inboundHandler(response http.ResponseWriter, request *http.Request) { - parsedEmail, err := Parse(request) + parsedEmail, err := ParseWithAttachments(request) if err != nil { log.Fatal(err) } fmt.Print(parsedEmail.Headers["From"]) - for filename, contents := range parsedEmail.Attachments { + for filename, contents := range parsedEmail.ParsedAttachments { // Do something with an attachment handleAttachment(filename, contents) } - for section, body := range parsedEmail.Body { - // Do something with the email body - handleEmail(body) + emailLines := strings.Split(parsedEmail.TextBody, "\n") + for section, body := range emailLines { + // Do something with the email lines } + // Twilio SendGrid needs a 200 OK response to stop POSTing response.WriteHeader(http.StatusOK) } +// example of uploading an attachment to s3 using the Go sdk-2 +func handleAttachment(parsedEmail *ParsedEmail) { + for _, contents := range parsedEmail.ParsedAttachments { + if _, err := sgh.Client.Upload(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &uploadPath, + Body: contents.File, + ContentType: aws.String(contents.ContentType), + } + } +} + func main() { http.HandleFunc("/inbound", inboundHandler) if err := http.ListenAndServe(":8000", nil); err != nil { diff --git a/helpers/inbound/inbound.go b/helpers/inbound/inbound.go index 80bda4ea..1db90ece 100644 --- a/helpers/inbound/inbound.go +++ b/helpers/inbound/inbound.go @@ -14,16 +14,21 @@ import ( // ParsedEmail defines a multipart parsed email // Body and Attachments are only populated if the Raw option is checked on the SendGrid inbound configuration and are named for backwards compatability type ParsedEmail struct { + // Header values are capitalized, such as From and To + Headers map[string]string // Please see https://docs.sendgrid.com/for-developers/parsing-email/setting-up-the-inbound-parse-webhook to see the available fields in the email headers // all fields listed there are available within the headers map except for text which lives in the TextBody field - Headers map[string]string + ParsedValues map[string]string // Primary email body parsed with \n. A common approach is to Split by the \n to bring every line of the email into a string array - TextBody string + TextBody string + + // attachemnts have been fully parsed to include the filename, size, content type and actual file for uploading or processing ParsedAttachments map[string]*EmailAttachment // Raw only Attachments map[string][]byte - Body map[string]string + // accessed with text/html and text/plain. text/plain is always parsed to the TextBody field + Body map[string]string rawRequest *http.Request rawValues map[string][]string @@ -38,10 +43,12 @@ type EmailAttachment struct { ContentType string `json:"type"` } -// Parse parses an email using Go's multipart parser and populates the headers, body +// Parse parses an email using Go's multipart parser and populates the headers, and body +// This method skips processing the attachment file and is therefore more performant func Parse(request *http.Request) (*ParsedEmail, error) { result := ParsedEmail{ Headers: make(map[string]string), + ParsedValues: make(map[string]string), ParsedAttachments: make(map[string]*EmailAttachment), Body: make(map[string]string), @@ -60,6 +67,7 @@ func ParseWithAttachments(request *http.Request) (*ParsedEmail, error) { result := ParsedEmail{ Headers: make(map[string]string), ParsedAttachments: make(map[string]*EmailAttachment), + ParsedValues: make(map[string]string), Body: make(map[string]string), Attachments: make(map[string][]byte), @@ -91,7 +99,7 @@ func (email *ParsedEmail) parse() error { } if len(v) > 0 { - email.Headers[k] = v[0] + email.ParsedValues[k] = v[0] } } From c886173770c802e19ade3464eab3bcfcdf021edf Mon Sep 17 00:00:00 2001 From: Quest Henkart Date: Sat, 9 Oct 2021 18:55:09 +0200 Subject: [PATCH 06/14] parse envelope --- helpers/inbound/inbound.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/helpers/inbound/inbound.go b/helpers/inbound/inbound.go index 1db90ece..b07d5c63 100644 --- a/helpers/inbound/inbound.go +++ b/helpers/inbound/inbound.go @@ -22,6 +22,12 @@ type ParsedEmail struct { // Primary email body parsed with \n. A common approach is to Split by the \n to bring every line of the email into a string array TextBody string + // Envelope expresses the exact email address that the email was addressed to and the exact email address it was from, without extra characters + Envelope struct { + From string `json:"from"` + To []string `json:"to"` + } + // attachemnts have been fully parsed to include the filename, size, content type and actual file for uploading or processing ParsedAttachments map[string]*EmailAttachment @@ -87,6 +93,13 @@ func (email *ParsedEmail) parse() error { email.rawValues = email.rawRequest.MultipartForm.Value + // unmarshal the envelope + if len(email.rawValues["envelope"]) > 0 { + if err := json.Unmarshal([]byte(email.rawValues["envelope"][0]), &email.Envelope); err != nil { + return err + } + } + // parse included headers if len(email.rawValues["headers"]) > 0 { email.parseHeaders(email.rawValues["headers"][0]) @@ -94,7 +107,7 @@ func (email *ParsedEmail) parse() error { // apply the rest of the SendGrid fields to the headers map for k, v := range email.rawValues { - if k == "text" || k == "email" { + if k == "text" || k == "email" || k == "headers" || k == "envelope" { continue } From 9f3fe8db3991e9bc71d005b9de806a4a9eb80500 Mon Sep 17 00:00:00 2001 From: Quest Henkart Date: Sat, 9 Oct 2021 18:56:41 +0200 Subject: [PATCH 07/14] update docs --- helpers/inbound/README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/helpers/inbound/README.md b/helpers/inbound/README.md index 683939e2..cd14b0f6 100644 --- a/helpers/inbound/README.md +++ b/helpers/inbound/README.md @@ -8,6 +8,10 @@ * [Contributing](#contributing) # Fields + +### parsedEmail.Envelope + To and From represent the exact email addresses that the email was sent to and the exact email address of the sender. There are no special characters and these fields are safe to use without further parsing as email addresses + ### parsedEmail.ParsedValues Please see [Send Grid Docs](https://docs.sendgrid.com/for-developers/parsing-email/setting-up-the-inbound-parse-webhook) to see what fields are available and preparsed by SendGrid. Use these fields over the Headers as they are parsed by SendGrid and gauranteed to be consistent @@ -40,7 +44,7 @@ func inboundHandler(response http.ResponseWriter, request *http.Request) { log.Fatal(err) } - fmt.Print(parsedEmail.Headers["From"]) + fmt.Print(parsedEmail.Envelope.From) for filename, contents := range parsedEmail.ParsedAttachments { // Do something with an attachment From c7d76435037ac29cb4f00e0eba8c09b754c363e5 Mon Sep 17 00:00:00 2001 From: Quest Henkart Date: Sat, 9 Oct 2021 19:25:51 +0200 Subject: [PATCH 08/14] docs --- helpers/inbound/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/inbound/README.md b/helpers/inbound/README.md index cd14b0f6..5b69d6e4 100644 --- a/helpers/inbound/README.md +++ b/helpers/inbound/README.md @@ -10,7 +10,7 @@ # Fields ### parsedEmail.Envelope - To and From represent the exact email addresses that the email was sent to and the exact email address of the sender. There are no special characters and these fields are safe to use without further parsing as email addresses + parsedEmail.Envelope.To and parsedEmail.Envelope.From represent the exact email addresses that the email was sent to and the exact email address of the sender. There are no special characters and these fields are safe to use without further parsing as email addresses ### parsedEmail.ParsedValues Please see [Send Grid Docs](https://docs.sendgrid.com/for-developers/parsing-email/setting-up-the-inbound-parse-webhook) to see what fields are available and preparsed by SendGrid. Use these fields over the Headers as they are parsed by SendGrid and gauranteed to be consistent From c2e0cd0e6fc50f58d5a7b3b032accb7e1212ae0e Mon Sep 17 00:00:00 2001 From: Quest Henkart Date: Sat, 9 Oct 2021 19:28:10 +0200 Subject: [PATCH 09/14] whitespace --- helpers/inbound/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/helpers/inbound/README.md b/helpers/inbound/README.md index 5b69d6e4..819f10e6 100644 --- a/helpers/inbound/README.md +++ b/helpers/inbound/README.md @@ -50,7 +50,8 @@ func inboundHandler(response http.ResponseWriter, request *http.Request) { // Do something with an attachment handleAttachment(filename, contents) } - + + emailLines := strings.Split(parsedEmail.TextBody, "\n") for section, body := range emailLines { // Do something with the email lines From 4b34574e1db5d2668d6bfb5be7887fb46407722d Mon Sep 17 00:00:00 2001 From: Quest Henkart Date: Sat, 9 Oct 2021 19:28:51 +0200 Subject: [PATCH 10/14] simplify --- helpers/inbound/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/helpers/inbound/README.md b/helpers/inbound/README.md index 819f10e6..9cc31f80 100644 --- a/helpers/inbound/README.md +++ b/helpers/inbound/README.md @@ -52,8 +52,7 @@ func inboundHandler(response http.ResponseWriter, request *http.Request) { } - emailLines := strings.Split(parsedEmail.TextBody, "\n") - for section, body := range emailLines { + for section, body := range strings.Split(parsedEmail.TextBody, "\n") { // Do something with the email lines } From f8317bd962d97e70f32b148ce7681a71cc8cc4f8 Mon Sep 17 00:00:00 2001 From: Quest Henkart Date: Sat, 9 Oct 2021 19:45:16 +0200 Subject: [PATCH 11/14] comments --- helpers/inbound/inbound.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/inbound/inbound.go b/helpers/inbound/inbound.go index b07d5c63..ae0e4268 100644 --- a/helpers/inbound/inbound.go +++ b/helpers/inbound/inbound.go @@ -14,7 +14,7 @@ import ( // ParsedEmail defines a multipart parsed email // Body and Attachments are only populated if the Raw option is checked on the SendGrid inbound configuration and are named for backwards compatability type ParsedEmail struct { - // Header values are capitalized, such as From and To + // Header values are raw and not pre-processed by SendGrid. They may change depending on the email client. Use carefully Headers map[string]string // Please see https://docs.sendgrid.com/for-developers/parsing-email/setting-up-the-inbound-parse-webhook to see the available fields in the email headers // all fields listed there are available within the headers map except for text which lives in the TextBody field From 889a4cd4bfd6ea8f9cfaf077a15abc528c448f11 Mon Sep 17 00:00:00 2001 From: Quest Henkart Date: Fri, 15 Oct 2021 18:45:15 +0200 Subject: [PATCH 12/14] defensively checks parsed header to avoid breaking email clients --- helpers/inbound/inbound.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/helpers/inbound/inbound.go b/helpers/inbound/inbound.go index ae0e4268..7849231b 100644 --- a/helpers/inbound/inbound.go +++ b/helpers/inbound/inbound.go @@ -229,6 +229,11 @@ func (email *ParsedEmail) parseHeaders(headers string) { splitHeaders := strings.Split(strings.TrimSpace(headers), "\n") for _, header := range splitHeaders { splitHeader := strings.SplitN(header, ": ", 2) + // keeps outlook emails from causing a panic + if len(splitHeader) != 2 { + continue + } + email.Headers[splitHeader[0]] = splitHeader[1] } } From 8afb27a9e6f10c73770bee3b011e0286dc46b27f Mon Sep 17 00:00:00 2001 From: Quest Henkart Date: Fri, 15 Oct 2021 18:50:19 +0200 Subject: [PATCH 13/14] updates warning in documentation --- helpers/inbound/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/inbound/README.md b/helpers/inbound/README.md index 9cc31f80..10f76f68 100644 --- a/helpers/inbound/README.md +++ b/helpers/inbound/README.md @@ -22,7 +22,7 @@ are populated *only* when the raw option is checked in the SendGrid Dashboard. However unless you need the raw HTML body, it is not necessary. The fields are named as they are for backward compatability ### parsedEmail.Headers - these may change depending on the email client and are not pre-parsed by SendGrid, use carefully + this field is deprecated. Use the SendGrid processed fields in ParsedValues instead. While it maintains its presence to avoid breaking changes, it provides raw, unprocessed headers and not all email clients are compatible. For example. these fields will be empty if the email cient is Outlook.com # Example Usage From 87a35f491be0bb4623a42cebc79a7eaa3a106a02 Mon Sep 17 00:00:00 2001 From: Quest Henkart Date: Mon, 24 Jan 2022 09:48:27 +0530 Subject: [PATCH 14/14] updates docs, DRY, and adds an additional test --- helpers/inbound/README.md | 38 +++++++++++++++++++++------------ helpers/inbound/inbound.go | 26 ++++++++++------------ helpers/inbound/inbound_test.go | 5 +++++ 3 files changed, 40 insertions(+), 29 deletions(-) diff --git a/helpers/inbound/README.md b/helpers/inbound/README.md index 10f76f68..9b26fc2f 100644 --- a/helpers/inbound/README.md +++ b/helpers/inbound/README.md @@ -9,20 +9,30 @@ # Fields -### parsedEmail.Envelope - parsedEmail.Envelope.To and parsedEmail.Envelope.From represent the exact email addresses that the email was sent to and the exact email address of the sender. There are no special characters and these fields are safe to use without further parsing as email addresses +### ParsedEmail.Envelope + ParsedEmail.Envelope.To and ParsedEmail.Envelope.From represent the exact email addresses that the email was sent to and the exact email address of the sender. There are no special characters and these fields are safe to use without further parsing as email addresses -### parsedEmail.ParsedValues - Please see [Send Grid Docs](https://docs.sendgrid.com/for-developers/parsing-email/setting-up-the-inbound-parse-webhook) to see what fields are available and preparsed by SendGrid. Use these fields over the Headers as they are parsed by SendGrid and gauranteed to be consistent +### ParsedEmail.ParsedValues + Please see [SendGrid Docs](https://docs.sendgrid.com/for-developers/parsing-email/setting-up-the-inbound-parse-webhook) to see what fields are available and preparsed by SendGrid. Use these fields over the Headers as they are parsed by SendGrid and gauranteed to be consistent -### parsedEmail.TextBody - this field will satisfy most cases. SendGrid pre-parses the body into a plain text string separated with \n +### ParsedEmail.TextBody + this field will satisfy most cases. SendGrid pre-parses the body into a plain text string separated with \n -### parsedEmail.Body and parsedEmail.Attachments - are populated *only* when the raw option is checked in the SendGrid Dashboard. However unless you need the raw HTML body, it is not necessary. The fields are named as they are for backward compatability +### ParsedEmail.ParsedAttachments + populated **only** when processing the email with ParseWithAttachments(). Provides the following ease of use values + - File: full attachment for uploading or processing (see example to upload to s3) + - Size: file size, useful for filtering or setting upper limits to attachments + - Filename: copies the original filename of the attachment, if there is not one, it defaults to 'Untitled' + - ContentType: the type of file -### parsedEmail.Headers - this field is deprecated. Use the SendGrid processed fields in ParsedValues instead. While it maintains its presence to avoid breaking changes, it provides raw, unprocessed headers and not all email clients are compatible. For example. these fields will be empty if the email cient is Outlook.com +### ParsedEmail.Body + populated *only* when the raw option is checked in the SendGrid Dashboard. Provides the raw HTML body of the email, unless you need to record the exact unparsed HTML payload from the email client, you should use the parsed fields instead. The field is named Body for backward compatability + +### ParsedEmail.Attachments + populated *only* when the raw option is checked in the SendGrid Dashboard. This field is deprecated. Use ParsedAttachments instead which does not require the Raw setting, and provides parsed values to use and process the attachments + +### ParsedEmail.Headers + this field is deprecated. Use the SendGrid processed fields in ParsedValues instead. While it maintains its presence to avoid breaking changes, it provides raw, unprocessed headers and not all email clients are compatible. For example. these fields will be empty if the email cient is Outlook.com # Example Usage @@ -43,20 +53,20 @@ func inboundHandler(response http.ResponseWriter, request *http.Request) { if err != nil { log.Fatal(err) } - + fmt.Print(parsedEmail.Envelope.From) - + for filename, contents := range parsedEmail.ParsedAttachments { // Do something with an attachment handleAttachment(filename, contents) } - + for section, body := range strings.Split(parsedEmail.TextBody, "\n") { // Do something with the email lines } - + // Twilio SendGrid needs a 200 OK response to stop POSTing response.WriteHeader(http.StatusOK) } diff --git a/helpers/inbound/inbound.go b/helpers/inbound/inbound.go index 7849231b..3c688fca 100644 --- a/helpers/inbound/inbound.go +++ b/helpers/inbound/inbound.go @@ -28,7 +28,7 @@ type ParsedEmail struct { To []string `json:"to"` } - // attachemnts have been fully parsed to include the filename, size, content type and actual file for uploading or processing + // Attachments have been fully parsed to include the filename, size, content type and actual file for uploading or processing ParsedAttachments map[string]*EmailAttachment // Raw only @@ -49,10 +49,8 @@ type EmailAttachment struct { ContentType string `json:"type"` } -// Parse parses an email using Go's multipart parser and populates the headers, and body -// This method skips processing the attachment file and is therefore more performant -func Parse(request *http.Request) (*ParsedEmail, error) { - result := ParsedEmail{ +func newParsedEmail(request *http.Request) ParsedEmail { + return ParsedEmail{ Headers: make(map[string]string), ParsedValues: make(map[string]string), ParsedAttachments: make(map[string]*EmailAttachment), @@ -63,6 +61,12 @@ func Parse(request *http.Request) (*ParsedEmail, error) { rawRequest: request, withAttachments: false, } +} + +// Parse parses an email using Go's multipart parser and populates the headers, and body +// This method skips processing the attachment file and is therefore more performant +func Parse(request *http.Request) (*ParsedEmail, error) { + result := newParsedEmail(request) err := result.parse() return &result, err @@ -70,16 +74,8 @@ func Parse(request *http.Request) (*ParsedEmail, error) { // ParseWithAttachments parses an email using Go's multipart parser and populates the headers, body and processes attachments func ParseWithAttachments(request *http.Request) (*ParsedEmail, error) { - result := ParsedEmail{ - Headers: make(map[string]string), - ParsedAttachments: make(map[string]*EmailAttachment), - ParsedValues: make(map[string]string), - - Body: make(map[string]string), - Attachments: make(map[string][]byte), - rawRequest: request, - withAttachments: true, - } + result := newParsedEmail(request) + result.withAttachments = true err := result.parse() return &result, err diff --git a/helpers/inbound/inbound_test.go b/helpers/inbound/inbound_test.go index 5b906409..d776f767 100644 --- a/helpers/inbound/inbound_test.go +++ b/helpers/inbound/inbound_test.go @@ -152,6 +152,11 @@ func TestValidate(t *testing.T) { values: map[string][]string{"dkim": {"pass", "pass", "pass"}, "SPF": {"pass", "fail", "pass"}}, expectedError: fmt.Errorf("SPF validation failed"), }, + { + name: "FailedSpfandDkim", + values: map[string][]string{"dkim": {"pass", "pass", "fail"}, "SPF": {"pass", "fail", "pass"}}, + expectedError: fmt.Errorf("DKIM validation failed"), + }, { name: "success", values: map[string][]string{"dkim": {"pass", "pass", "pass"}, "SPF": {"pass", "pass", "pass"}},