Skip to content

Commit

Permalink
resolve host detection using url
Browse files Browse the repository at this point in the history
  • Loading branch information
@dduzgun-security
dduzgun-security committed Jan 10, 2025
1 parent 042c9f5 commit bb6ad0a
Show file tree
Hide file tree
Showing 4 changed files with 49 additions and 216 deletions.
121 changes: 11 additions & 110 deletions detect_gcs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,7 @@ package getter
import (
"fmt"
"net/url"
"path"
"regexp"
"strings"
"unicode"
)

// GCSDetector implements Detector to detect GCS URLs and turn
Expand All @@ -21,39 +18,32 @@ func (d *GCSDetector) Detect(src, _ string) (string, bool, error) {
return "", false, nil
}

if strings.Contains(src, ".googleapis.com/") {
return d.detectHTTP(src)
if !strings.HasPrefix(src, "http://") && !strings.HasPrefix(src, "https://") {
src = "https://" + src
}

parsedURL, err := url.Parse(src)
if err != nil {
return "", false, fmt.Errorf("error parsing GCS URL")
}

if strings.HasSuffix(parsedURL.Host, ".googleapis.com") {
return d.detectHTTP(strings.ReplaceAll(src, "https://", ""))
}

return "", false, nil
}

func (d *GCSDetector) detectHTTP(src string) (string, bool, error) {
src = path.Clean(src)

parts := strings.Split(src, "/")
if len(parts) < 5 {
return "", false, fmt.Errorf(
"URL is not a valid GCS URL")
}

version := parts[2]
if !isValidGCSVersion(version) {
return "", false, fmt.Errorf(
"GCS URL version is not valid")
}

bucket := parts[3]
if !isValidGCSBucketName(bucket) {
return "", false, fmt.Errorf(
"GCS URL bucket name is not valid")
}

object := strings.Join(parts[4:], "/")
if !isValidGCSObjectName(object) {
return "", false, fmt.Errorf(
"GCS URL object name is not valid")
}

url, err := url.Parse(fmt.Sprintf("https://www.googleapis.com/storage/%s/%s/%s",
version, bucket, object))
Expand All @@ -63,92 +53,3 @@ func (d *GCSDetector) detectHTTP(src string) (string, bool, error) {

return "gcs::" + url.String(), true, nil
}

func isValidGCSVersion(version string) bool {
versionPattern := `^v\d+$`
if matched, _ := regexp.MatchString(versionPattern, version); !matched {
return false
}
return true
}

// Validate the bucket name using the following rules: https://cloud.google.com/storage/docs/naming-buckets
func isValidGCSBucketName(bucket string) bool {
// Rule 1: Must be between 3 and 63 characters (or up to 222 if it contains dots, each component up to 63 chars)
if len(bucket) < 3 || len(bucket) > 63 {
if len(bucket) > 63 && len(bucket) <= 222 {
// If it contains dots, each segment between dots must be <= 63 chars
components := strings.Split(bucket, ".")
for _, component := range components {
if len(component) > 63 {
return false
}
}
} else {
return false
}
}

// Rule 2: Bucket name cannot start or end with a hyphen, dot, or underscore
if bucket[0] == '-' || bucket[0] == '.' || bucket[len(bucket)-1] == '-' || bucket[len(bucket)-1] == '.' || bucket[len(bucket)-1] == '_' {
return false
}

// Rule 3: Bucket name cannot contain spaces
if strings.Contains(bucket, " ") {
return false
}

// Rule 4: Bucket name cannot be an IP address (only digits and dots, e.g., 192.168.5.4)
ipPattern := `^(\d{1,3}\.){3}\d{1,3}$`
if matched, _ := regexp.MatchString(ipPattern, bucket); matched {
return false
}

// Rule 5: Bucket name cannot start with "goog"
if strings.HasPrefix(bucket, "goog") {
return false
}

// Rule 6: Bucket name cannot contain "google" or common misspellings like "g00gle"
googlePattern := `google|g00gle`
if matched, _ := regexp.MatchString(googlePattern, bucket); matched {
return false
}

// Rule 7: Bucket name can only contain lowercase letters, digits, dashes, underscores, and dots
bucketPattern := `^[a-z0-9\-_\.]+$`
if matched, _ := regexp.MatchString(bucketPattern, bucket); !matched {
return false
}

return true
}

// Validate the object name using the following rules: https://cloud.google.com/storage/docs/naming-objects
func isValidGCSObjectName(object string) bool {
// Rule 1: Object names cannot contain Carriage Return (\r) or Line Feed (\n) characters
if strings.Contains(object, "\r") || strings.Contains(object, "\n") {
return false
}

// Rule 2: Object names cannot start with '.well-known/acme-challenge/'
if strings.HasPrefix(object, ".well-known/acme-challenge/") {
return false
}

// Rule 3: Object names cannot be exactly '.' or '..'
if object == "." || object == ".." {
return false
}

// Rule 4: Ensure that the object name contains only valid Unicode characters
// (for simplicity, let's ensure it's not empty and does not contain any forbidden control characters)
for _, r := range object {
if !unicode.IsPrint(r) && !unicode.IsSpace(r) && r != '.' && r != '-' && r != '/' {
return false
}
}

return true
}
107 changes: 3 additions & 104 deletions detect_gcs_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,31 +73,8 @@ func TestGCSDetector_MalformedDetectHTTP(t *testing.T) {
"",
},
{
"not valid version",
"www.googleapis.com/storage/invalid-version/my-bucket/foo",
"GCS URL version is not valid",
"",
},
{
"not valid bucket",
"www.googleapis.com/storage/v1/127.0.0.1/foo",
"GCS URL bucket name is not valid",
"",
},
{
"not valid object",
"www.googleapis.com/storage/v1/my-bucket/.well-known/acme-challenge/foo",
"GCS URL object name is not valid",
"",
},
{
"path traversal on bucket",
"www.googleapis.com/storage/v1/../foo/bar",
"URL is not a valid GCS URL",
"",
}, {
"path traversal on object",
"www.googleapis.com/storage/v1/my-bucket/../../../foo/bar",
"not valid url length",
"www.invalid.com/storage/v1",
"URL is not a valid GCS URL",
"",
},
Expand All @@ -114,85 +91,7 @@ func TestGCSDetector_MalformedDetectHTTP(t *testing.T) {
}

if output != tc.Output {
t.Fatalf("expected %s, got %s", tc.Output, output)
}
}
}

func TestIsValidGCSVersion(t *testing.T) {
cases := []struct {
Name string
Input string
Expected bool
}{
{
"valid version",
"v1",
true,
},
{
"invalid version",
"invalid1",
false,
},
}

for _, tc := range cases {
output := isValidGCSVersion(tc.Input)
if output != tc.Expected {
t.Fatalf("expected %t, got %t for test %s", tc.Expected, output, tc.Name)
}
}
}

func TestIsValidGCSBucketName(t *testing.T) {
cases := []struct {
Name string
Input string
Expected bool
}{
{
"valid bucket name",
"my-bucket",
true,
},
{
"invalid bucket name",
"..",
false,
},
}

for _, tc := range cases {
output := isValidGCSBucketName(tc.Input)
if output != tc.Expected {
t.Fatalf("expected %t, got %t for test %s", tc.Expected, output, tc.Name)
}
}
}

func TestIsValidGCSObjectName(t *testing.T) {
cases := []struct {
Name string
Input string
Expected bool
}{
{
"valid object name",
"my-object",
true,
},
{
"invalid object name",
"..",
false,
},
}

for _, tc := range cases {
output := isValidGCSObjectName(tc.Input)
if output != tc.Expected {
t.Fatalf("expected %t, got %t for test %s", tc.Expected, output, tc.Name)
t.Fatalf("expected %s, got %s for %s", tc.Output, output, tc.Name)
}
}
}
13 changes: 11 additions & 2 deletions detect_s3.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,17 @@ func (d *S3Detector) Detect(src, _ string) (string, bool, error) {
return "", false, nil
}

if strings.Contains(src, ".amazonaws.com/") {
return d.detectHTTP(src)
if !strings.HasPrefix(src, "http://") && !strings.HasPrefix(src, "https://") {
src = "https://" + src
}

parsedURL, err := url.Parse(src)
if err != nil {
return "", false, fmt.Errorf("error parsing S3 URL")
}

if strings.HasSuffix(parsedURL.Host, ".amazonaws.com") {
return d.detectHTTP(strings.ReplaceAll(src, "https://", ""))
}

return "", false, nil
Expand Down
24 changes: 24 additions & 0 deletions get_gcs_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,30 @@ func TestGCSGetter_GetFile_OAuthAccessToken(t *testing.T) {
assertContents(t, dst, "# Main\n")
}

func Test_GCSGetter_ParseUrl(t *testing.T) {
tests := []struct {
name string
url string
}{
{
name: "valid host",
url: "https://www.googleapis.com/storage/v1/hc-go-getter-test/go-getter/foobar",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
g := new(GCSGetter)
u, err := url.Parse(tt.url)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
_, _, _, err = g.parseURL(u)
if err != nil {
t.Fatalf("wasn't expecting error, got %s", err)
}
})
}
}
func Test_GCSGetter_ParseUrl_Malformed(t *testing.T) {
tests := []struct {
name string
Expand Down

0 comments on commit bb6ad0a

Please sign in to comment.