Support a separate URL base for pre-signed URLs

This allows the Ark server to use one URL for the majority of
communications with S3 (or compatible) object storage, and a different
URL base for pre-signed URLs (for streaming logs, etc. to clients).

Signed-off-by: Andy Goldstein <[email protected]>

Author: ncdc

docs/api-types/backupstoragelocation.md

@@ -49,6 +49,7 @@ The configurable parameters are as follows:
 | `region` | string | Empty | *Example*: "us-east-1"<br><br>See [AWS documentation][3] for the full list.<br><br>Queried from the AWS S3 API if not provided. |
 | `s3ForcePathStyle` | bool | `false` | Set this to `true` if you are using a local storage service like Minio. |
 | `s3Url` | string | Required field for non-AWS-hosted storage| *Example*: http://minio:9000<br><br>You can specify the AWS S3 URL here for explicitness, but Ark can already generate it from `region`, and `bucket`. This field is primarily for local storage services like Minio.|
+| `publicUrl` | string | Empty | *Example*: https://minio.mycluster.com<br><br>If specified, use this instead of `s3Url` when generating download URLs (e.g., for logs). This field is primarily for local storage services like Minio.|
 | `kmsKeyId` | string | Empty | *Example*: "502b409c-4da1-419f-a16e-eif453b3i49f" or "alias/`<KMS-Key-Alias-Name>`"<br><br>Specify an [AWS KMS key][10] id or alias to enable encryption of the backups stored in S3. Only works with AWS S3 and may require explicitly granting key usage rights.|
 
 #### Azure
@@ -67,4 +68,4 @@ No parameters required.
 [0]: #aws
 [1]: #gcp
 [2]: #azure
-[3]: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions
+[3]: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions

pkg/cloudprovider/aws/object_store.go

@@ -34,6 +34,7 @@ import (
 
 const (
 	s3URLKey            = "s3Url"
+	publicURLKey        = "publicUrl"
 	kmsKeyIDKey         = "kmsKeyId"
 	s3ForcePathStyleKey = "s3ForcePathStyle"
 	bucketKey           = "bucket"
@@ -42,6 +43,7 @@ const (
 type objectStore struct {
 	log        logrus.FieldLogger
 	s3         *s3.S3
+	preSignS3  *s3.S3
 	s3Uploader *s3manager.Uploader
 	kmsKeyID   string
 }
@@ -54,6 +56,7 @@ func (o *objectStore) Init(config map[string]string) error {
 	var (
 		region              = config[regionKey]
 		s3URL               = config[s3URLKey]
+		publicURL           = config[publicURLKey]
 		kmsKeyID            = config[kmsKeyIDKey]
 		s3ForcePathStyleVal = config[s3ForcePathStyleKey]
 
@@ -83,20 +86,52 @@ func (o *objectStore) Init(config map[string]string) error {
 		}
 	}
 
+	serverConfig, err := newAWSConfig(s3URL, region, s3ForcePathStyle)
+	if err != nil {
+		return err
+	}
+
+	serverSession, err := getSession(serverConfig)
+	if err != nil {
+		return err
+	}
+
+	o.s3 = s3.New(serverSession)
+	o.s3Uploader = s3manager.NewUploader(serverSession)
+	o.kmsKeyID = kmsKeyID
+
+	if publicURL != "" {
+		publicConfig, err := newAWSConfig(publicURL, region, s3ForcePathStyle)
+		if err != nil {
+			return err
+		}
+		publicSession, err := getSession(publicConfig)
+		if err != nil {
+			return err
+		}
+		o.preSignS3 = s3.New(publicSession)
+	} else {
+		o.preSignS3 = o.s3
+	}
+
+	return nil
+}
+
+func newAWSConfig(url, region string, forcePathStyle bool) (*aws.Config, error) {
 	awsConfig := aws.NewConfig().
 		WithRegion(region).
-		WithS3ForcePathStyle(s3ForcePathStyle)
+		WithS3ForcePathStyle(forcePathStyle)
 
-	if s3URL != "" {
-		if !IsValidS3URLScheme(s3URL) {
-			return errors.Errorf("Invalid s3Url: %s", s3URL)
+	if url != "" {
+		if !IsValidS3URLScheme(url) {
+			return nil, errors.Errorf("Invalid s3 url: %s", url)
 		}
 
 		awsConfig = awsConfig.WithEndpointResolver(
 			endpoints.ResolverFunc(func(service, region string, optFns ...func(*endpoints.Options)) (endpoints.ResolvedEndpoint, error) {
 				if service == endpoints.S3ServiceID {
 					return endpoints.ResolvedEndpoint{
-						URL: s3URL,
+						URL: url,
 					}, nil
 				}
 
@@ -105,16 +140,7 @@ func (o *objectStore) Init(config map[string]string) error {
 		)
 	}
 
-	sess, err := getSession(awsConfig)
-	if err != nil {
-		return err
-	}
-
-	o.s3 = s3.New(sess)
-	o.s3Uploader = s3manager.NewUploader(sess)
-	o.kmsKeyID = kmsKeyID
-
-	return nil
+	return awsConfig, nil
 }
 
 func (o *objectStore) PutObject(bucket, key string, body io.Reader) error {
@@ -208,7 +234,7 @@ func (o *objectStore) DeleteObject(bucket, key string) error {
 }
 
 func (o *objectStore) CreateSignedURL(bucket, key string, ttl time.Duration) (string, error) {
-	req, _ := o.s3.GetObjectRequest(&s3.GetObjectInput{
+	req, _ := o.preSignS3.GetObjectRequest(&s3.GetObjectInput{
 		Bucket: aws.String(bucket),
 		Key:    aws.String(key),
 	})