From 10cc3a988d4593506ad7357dff7a11ac29e6f78a Mon Sep 17 00:00:00 2001 From: skotambkar Date: Fri, 22 Jan 2021 22:36:58 -0800 Subject: [PATCH 1/9] add getter/setter for endpoint source on context --- aws/middleware/metadata.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/aws/middleware/metadata.go b/aws/middleware/metadata.go index f8e6a98f72d..28201217406 100644 --- a/aws/middleware/metadata.go +++ b/aws/middleware/metadata.go @@ -2,6 +2,7 @@ package middleware import ( "context" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/smithy-go/middleware" ) @@ -150,3 +151,17 @@ func setOperationName(ctx context.Context, value string) context.Context { func SetPartitionID(ctx context.Context, value string) context.Context { return middleware.WithStackValue(ctx, partitionIDKey{}, value) } + +// EndpointSource key +type endpointSourceKey struct{} + +// GetEndpointSource returns an endpoint source if set on context +func GetEndpointSource(ctx context.Context) (v aws.EndpointSource) { + v, _ = middleware.GetStackValue(ctx, endpointSourceKey{}).(aws.EndpointSource) + return v +} + +// SetEndpointSource sets endpoint source on context +func SetEndpointSource(ctx context.Context, value aws.EndpointSource) context.Context { + return middleware.WithStackValue(ctx, endpointSourceKey{}, value) +} From 14e884b9a73a7fc157202b975cc09a3a1b6b08f2 Mon Sep 17 00:00:00 2001 From: skotambkar Date: Fri, 22 Jan 2021 22:37:12 -0800 Subject: [PATCH 2/9] codegen setter for endpoint source when resolving endpoint --- .../amazon/smithy/aws/go/codegen/EndpointGenerator.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/EndpointGenerator.java b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/EndpointGenerator.java index 67780b669be..ba7ee133140 100644 --- a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/EndpointGenerator.java +++ b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/EndpointGenerator.java @@ -297,6 +297,10 @@ private void generateMiddlewareResolverBody(GoStackStepMiddlewareGenerator g, Go }); w.write("ctx = awsmiddleware.SetSigningName(ctx, signingName)"); }); + + // set endoint source on context + w.write("ctx = awsmiddleware.SetEndpointSource(ctx, endpoint.Source)"); + // set host-name immutable on context w.write("ctx = smithyhttp.SetHostnameImmutable(ctx, endpoint.HostnameImmutable)"); // set signing region on context w.write("ctx = awsmiddleware.SetSigningRegion(ctx, endpoint.SigningRegion)"); From 9da753e137f94c2867c01e456e776eb724fda9f8 Mon Sep 17 00:00:00 2001 From: skotambkar Date: Fri, 22 Jan 2021 22:37:43 -0800 Subject: [PATCH 3/9] generation changes for s3 and s3control client --- service/s3/endpoints.go | 1 + service/s3control/endpoints.go | 1 + 2 files changed, 2 insertions(+) diff --git a/service/s3/endpoints.go b/service/s3/endpoints.go index e8030225647..4652db06f87 100644 --- a/service/s3/endpoints.go +++ b/service/s3/endpoints.go @@ -105,6 +105,7 @@ func (m *ResolveEndpoint) HandleSerialize(ctx context.Context, in middleware.Ser } ctx = awsmiddleware.SetSigningName(ctx, signingName) } + ctx = awsmiddleware.SetEndpointSource(ctx, endpoint.Source) ctx = smithyhttp.SetHostnameImmutable(ctx, endpoint.HostnameImmutable) ctx = awsmiddleware.SetSigningRegion(ctx, endpoint.SigningRegion) ctx = awsmiddleware.SetPartitionID(ctx, endpoint.PartitionID) diff --git a/service/s3control/endpoints.go b/service/s3control/endpoints.go index 8ccffa080f9..7889cc861f7 100644 --- a/service/s3control/endpoints.go +++ b/service/s3control/endpoints.go @@ -105,6 +105,7 @@ func (m *ResolveEndpoint) HandleSerialize(ctx context.Context, in middleware.Ser } ctx = awsmiddleware.SetSigningName(ctx, signingName) } + ctx = awsmiddleware.SetEndpointSource(ctx, endpoint.Source) ctx = smithyhttp.SetHostnameImmutable(ctx, endpoint.HostnameImmutable) ctx = awsmiddleware.SetSigningRegion(ctx, endpoint.SigningRegion) ctx = awsmiddleware.SetPartitionID(ctx, endpoint.PartitionID) From 7e464f99a104da195de769576aa0ec634b0cdec7 Mon Sep 17 00:00:00 2001 From: skotambkar Date: Fri, 22 Jan 2021 22:40:03 -0800 Subject: [PATCH 4/9] update cross partition error handling to not return error, when resolving partiiton is not possible --- service/internal/s3shared/resource_request.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/service/internal/s3shared/resource_request.go b/service/internal/s3shared/resource_request.go index e940b7f368a..2d8a953581e 100644 --- a/service/internal/s3shared/resource_request.go +++ b/service/internal/s3shared/resource_request.go @@ -45,12 +45,13 @@ func (r ResourceRequest) AllowCrossRegion() bool { } // IsCrossPartition returns true if request is configured for region of another partition, than -// the partition that resource ARN region resolves to. +// the partition that resource ARN region resolves to. IsCrossPartition will not return an error, +// if request is not configured with a specific partition id. This might happen if customer provides +// custom endpoint url, but does not associate a partition id with it. func (r ResourceRequest) IsCrossPartition() (bool, error) { - // These error checks should never be triggered, unless validations are turned off rv := r.PartitionID if len(rv) == 0 { - return false, fmt.Errorf("partition id was not found for provided request region") + return false, nil } av := r.Resource.GetARN().Partition From f7032a96370529dae94621dce3d98ef18304ca3c Mon Sep 17 00:00:00 2001 From: skotambkar Date: Fri, 22 Jan 2021 22:44:17 -0800 Subject: [PATCH 5/9] update arn processing customization to not prefix labels in case of custom endpoint source. Also update outpost id processing customization to not prefix labels in case of custom endpoint source. This change is required for handling s3 control special cases wrt createAccesspoint, createBucket and ListregionalBuckets operations. --- .../internal/customizations/process_arn_resource.go | 3 ++- .../internal/customizations/process_outpost_id.go | 12 +++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/service/s3control/internal/customizations/process_arn_resource.go b/service/s3control/internal/customizations/process_arn_resource.go index 23491b6e709..4ef83545afd 100644 --- a/service/s3control/internal/customizations/process_arn_resource.go +++ b/service/s3control/internal/customizations/process_arn_resource.go @@ -273,7 +273,8 @@ func buildOutpostAccessPointRequest(ctx context.Context, options outpostAccessPo // resolve regional endpoint for resolved region. var endpoint aws.Endpoint var err error - if endpointsID == "s3" { + endpointSource := awsmiddleware.GetEndpointSource(ctx) + if endpointsID == "s3" && endpointSource != aws.EndpointSourceCustom { // use s3 endpoint resolver endpoint, err = s3endpoints.New().ResolveEndpoint(resolveRegion, s3endpoints.Options{ DisableHTTPS: options.EndpointResolverOptions.DisableHTTPS, diff --git a/service/s3control/internal/customizations/process_outpost_id.go b/service/s3control/internal/customizations/process_outpost_id.go index aef5537ef19..49891b02ef8 100644 --- a/service/s3control/internal/customizations/process_outpost_id.go +++ b/service/s3control/internal/customizations/process_outpost_id.go @@ -8,6 +8,7 @@ import ( "github.com/aws/smithy-go/middleware" smithyhttp "github.com/aws/smithy-go/transport/http" + "github.com/aws/aws-sdk-go-v2/aws" awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" "github.com/aws/aws-sdk-go-v2/service/internal/s3shared" ) @@ -55,7 +56,6 @@ func (m *processOutpostIDMiddleware) HandleSerialize( return out, metadata, fmt.Errorf("unknown request type %T", req) } - serviceEndpointLabel := "s3-outposts." requestRegion := awsmiddleware.GetRegion(ctx) // validate if fips @@ -67,8 +67,14 @@ func (m *processOutpostIDMiddleware) HandleSerialize( return out, metadata, fmt.Errorf("dualstack is not supported for outposts request") } - // set request url - req.URL.Host = serviceEndpointLabel + requestRegion + ".amazonaws.com" + // if endpoint source is a custom endpoint source, do not modify host prefix. + // This is a requirement for s3 vpc interface endpoints. + if v := awsmiddleware.GetEndpointSource(ctx); v != aws.EndpointSourceCustom { + serviceEndpointLabel := "s3-outposts." + + // set request url + req.URL.Host = serviceEndpointLabel + requestRegion + ".amazonaws.com" + } // Disable endpoint host prefix for s3-control ctx = smithyhttp.DisableEndpointHostPrefix(ctx, true) From 3ce33d9351e7942a183b44ffcf55445d32bdf2cc Mon Sep 17 00:00:00 2001 From: skotambkar Date: Fri, 22 Jan 2021 22:56:21 -0800 Subject: [PATCH 6/9] add unit test to verify vpc endpoint interface support for s3 and s3control --- .../customizations/update_endpoint_test.go | 288 ++++++++++++++---- .../customizations/update_endpoint_test.go | 260 ++++++++++++---- 2 files changed, 428 insertions(+), 120 deletions(-) diff --git a/service/s3/internal/customizations/update_endpoint_test.go b/service/s3/internal/customizations/update_endpoint_test.go index b93755a7c55..a13ca15edb7 100644 --- a/service/s3/internal/customizations/update_endpoint_test.go +++ b/service/s3/internal/customizations/update_endpoint_test.go @@ -7,17 +7,15 @@ import ( "strings" "testing" + "github.com/aws/smithy-go/middleware" "github.com/aws/smithy-go/ptr" + smithyhttp "github.com/aws/smithy-go/transport/http" "github.com/aws/aws-sdk-go-v2/aws" awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" "github.com/aws/aws-sdk-go-v2/internal/awstesting/unit" - "github.com/aws/aws-sdk-go-v2/service/s3/internal/endpoints" - - "github.com/aws/smithy-go/middleware" - smithyhttp "github.com/aws/smithy-go/transport/http" - "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/internal/endpoints" ) type s3BucketTest struct { @@ -240,14 +238,7 @@ func TestUpdateEndpointBuild(t *testing.T) { func TestEndpointWithARN(t *testing.T) { // test cases - cases := map[string]struct { - options s3.Options - bucket string - expectedErr string - expectedReqURL string - expectedSigningName string - expectedSigningRegion string - }{ + cases := map[string]testCaseForEndpointCustomization{ "Outpost AccessPoint with no S3UseARNRegion flag set": { bucket: "arn:aws:s3-outposts:us-west-2:123456789012:outpost:op-01234567890123456:accesspoint:myaccesspoint", options: s3.Options{ @@ -503,7 +494,9 @@ func TestEndpointWithARN(t *testing.T) { return aws.Endpoint{}, nil }), }, - expectedErr: "partition id was not found for provided request region", + expectedReqURL: "https://myendpoint-123456789012.s3-accesspoint.us-west-2.amazonaws.com/testkey?x-id=GetObject", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", }, "Custom Resolver Without PartitionID in Cross-Region Target": { bucket: "arn:aws:s3:us-west-2:123456789012:accesspoint:myendpoint", @@ -531,7 +524,9 @@ func TestEndpointWithARN(t *testing.T) { return aws.Endpoint{}, nil }), }, - expectedErr: "partition id was not found for provided request region", + expectedReqURL: "https://myendpoint-123456789012.s3-accesspoint.us-west-2.amazonaws.com/testkey?x-id=GetObject", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", }, "bucket host-style": { bucket: "mock-bucket", @@ -618,64 +613,227 @@ func TestEndpointWithARN(t *testing.T) { for name, c := range cases { t.Run(name, func(t *testing.T) { + runValidations(t, c, func(ctx context.Context, svc *s3.Client, fm *requestRetrieverMiddleware) (interface{}, error) { + return svc.GetObject(ctx, &s3.GetObjectInput{ + Bucket: ptr.String(c.bucket), + Key: ptr.String("testkey"), + }, addRequestRetriever(fm)) + }) + }) + } +} - // options - opts := c.options.Copy() - opts.Credentials = unit.StubCredentialsProvider{} - opts.HTTPClient = smithyhttp.NopClient{} - opts.Retryer = aws.NopRetryer{} - - // build an s3 client - svc := s3.New(opts) - // setup a request retriever middleware - fm := requestRetrieverMiddleware{} - - ctx := context.Background() - - // call an operation - _, err := svc.GetObject(ctx, &s3.GetObjectInput{ - Bucket: ptr.String(c.bucket), - Key: ptr.String("testkey"), - }, func(options *s3.Options) { - // append request retriever middleware for request inspection - options.APIOptions = append(options.APIOptions, - func(stack *middleware.Stack) error { - // adds AFTER operation serializer middleware - stack.Serialize.Insert(&fm, "OperationSerializer", middleware.After) - return nil - }) +type testCaseForEndpointCustomization struct { + options s3.Options + bucket string + operation func(ctx context.Context, svc *s3.Client, fm *requestRetrieverMiddleware) (interface{}, error) + expectedErr string + expectedReqURL string + expectedSigningName string + expectedSigningRegion string +} + +var addRequestRetriever = func(fm *requestRetrieverMiddleware) func(options *s3.Options) { + return func(options *s3.Options) { + // append request retriever middleware for request inspection + options.APIOptions = append(options.APIOptions, + func(stack *middleware.Stack) error { + // adds AFTER operation serializer middleware + stack.Serialize.Insert(fm, "OperationSerializer", middleware.After) + return nil }) + } +} - // inspect any errors - if len(c.expectedErr) != 0 { - if err == nil { - t.Fatalf("expected error, got none") - } - if a, e := err.Error(), c.expectedErr; !strings.Contains(a, e) { - t.Fatalf("expect error code to contain %q, got %q", e, a) +func TestVPC_CustomEndpoint(t *testing.T) { + cases := map[string]testCaseForEndpointCustomization{ + "standard custom endpoint url": { + bucket: "bucketname", + options: s3.Options{ + EndpointResolver: s3.EndpointResolverFromURL("https://beta.example.com", func(endpoint *aws.Endpoint) { + endpoint.SigningRegion = "us-west-2" + }), + Region: "us-west-2", + }, + expectedReqURL: "https://bucketname.beta.example.com/", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", + }, + "AccessPoint with custom endpoint url": { + bucket: "arn:aws:s3:us-west-2:123456789012:accesspoint:myendpoint", + options: s3.Options{ + EndpointResolver: s3.EndpointResolverFromURL("https://beta.example.com", func(endpoint *aws.Endpoint) { + endpoint.SigningRegion = "us-west-2" + }), + Region: "us-west-2", + }, + expectedReqURL: "https://myendpoint-123456789012.beta.example.com/", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", + }, + "Outpost AccessPoint with custom endpoint url": { + bucket: "arn:aws:s3-outposts:us-west-2:123456789012:outpost:op-01234567890123456:accesspoint:myaccesspoint", + options: s3.Options{ + EndpointResolver: s3.EndpointResolverFromURL("https://beta.example.com", func(endpoint *aws.Endpoint) { + endpoint.SigningRegion = "us-west-2" + }), + Region: "us-west-2", + }, + expectedReqURL: "https://myaccesspoint-123456789012.op-01234567890123456.beta.example.com/", + expectedSigningName: "s3-outposts", + expectedSigningRegion: "us-west-2", + }, + "ListBucket with custom endpoint url": { + options: s3.Options{ + EndpointResolver: s3.EndpointResolverFromURL("https://bucket.vpce-123-abc.s3.us-west-2.vpce.amazonaws.com", func(endpoint *aws.Endpoint) { + endpoint.SigningRegion = "us-west-2" + }), + Region: "us-west-2", + }, + operation: func(ctx context.Context, svc *s3.Client, fm *requestRetrieverMiddleware) (interface{}, error) { + return svc.ListBuckets(ctx, &s3.ListBucketsInput{}, addRequestRetriever(fm)) + }, + expectedReqURL: "https://bucket.vpce-123-abc.s3.us-west-2.vpce.amazonaws.com/", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", + }, + "Path-style addressing with custom endpoint url": { + bucket: "bucketname", + options: s3.Options{ + EndpointResolver: s3.EndpointResolverFromURL("https://bucket.vpce-123-abc.s3.us-west-2.vpce.amazonaws.com", func(endpoint *aws.Endpoint) { + endpoint.SigningRegion = "us-west-2" + }), + Region: "us-west-2", + UsePathStyle: true, + }, + expectedReqURL: "https://bucket.vpce-123-abc.s3.us-west-2.vpce.amazonaws.com/bucketname", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", + }, + "Virtual host addressing with custom endpoint url": { + bucket: "bucketname", + options: s3.Options{ + EndpointResolver: s3.EndpointResolverFromURL("https://bucket.vpce-123-abc.s3.us-west-2.vpce.amazonaws.com", func(endpoint *aws.Endpoint) { + endpoint.SigningRegion = "us-west-2" + }), + Region: "us-west-2", + }, + expectedReqURL: "https://bucketname.bucket.vpce-123-abc.s3.us-west-2.vpce.amazonaws.com/", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", + }, + "Access-point with custom endpoint url and use_arn_region set": { + bucket: "arn:aws:s3:us-west-2:123456789012:accesspoint:myendpoint", + options: s3.Options{ + EndpointResolver: s3.EndpointResolverFromURL("https://accesspoint.vpce-123-abc.s3.us-west-2.vpce.amazonaws.com", func(endpoint *aws.Endpoint) { + endpoint.SigningRegion = "us-west-2" + }), + Region: "eu-west-1", + UseARNRegion: true, + }, + expectedReqURL: "https://myendpoint-123456789012.accesspoint.vpce-123-abc.s3.us-west-2.vpce.amazonaws.com/", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", + }, + "Custom endpoint url with Dualstack": { + bucket: "bucketname", + options: s3.Options{ + EndpointResolver: s3.EndpointResolverFromURL("https://beta.example.com", func(endpoint *aws.Endpoint) { + endpoint.SigningRegion = "us-west-2" + }), + Region: "us-west-2", + UseDualstack: true, + }, + expectedReqURL: "https://bucketname.beta.example.com/", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", + }, + "Outpost with custom endpoint url and Dualstack": { + bucket: "arn:aws:s3-outposts:us-west-2:123456789012:outpost:op-01234567890123456:accesspoint:myaccesspoint", + options: s3.Options{ + EndpointResolver: s3.EndpointResolverFromURL("https://beta.example.com", func(endpoint *aws.Endpoint) { + endpoint.SigningRegion = "us-west-2" + }), + Region: "us-west-2", + UseDualstack: true, + }, + expectedErr: "client configured for S3 Dual-stack but is not supported with resource ARN", + }, + "Standard custom endpoint url with Immutable Host": { + bucket: "bucketname", + options: s3.Options{ + EndpointResolver: s3.EndpointResolverFromURL("https://beta.example.com", func(endpoint *aws.Endpoint) { + endpoint.SigningRegion = "us-west-2" + endpoint.HostnameImmutable = true + }), + Region: "us-west-2", + }, + expectedReqURL: "https://beta.example.com/bucketname", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + runValidations(t, c, func(ctx context.Context, svc *s3.Client, fm *requestRetrieverMiddleware) (interface{}, error) { + if c.operation != nil { + return c.operation(ctx, svc, fm) } - return - } - if err != nil { - t.Fatalf("expect no error, got %v", err) - } - // build the captured request - req := fm.request.Build(ctx) - // verify the built request is as expected - if e, a := c.expectedReqURL, req.URL.String(); e != a { - t.Fatalf("expect url %s, got %s", e, a) - } + return svc.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: ptr.String(c.bucket), + }, addRequestRetriever(fm)) + }) + }) + } +} - if e, a := c.expectedSigningRegion, fm.signingRegion; !strings.EqualFold(e, a) { - t.Fatalf("expect signing region as %s, got %s", e, a) - } +func runValidations(t *testing.T, c testCaseForEndpointCustomization, operation func( + context.Context, *s3.Client, *requestRetrieverMiddleware) (interface{}, error)) { + // options + opts := c.options.Copy() + opts.Credentials = unit.StubCredentialsProvider{} + opts.HTTPClient = smithyhttp.NopClient{} + opts.Retryer = aws.NopRetryer{} + + // build an s3 client + svc := s3.New(opts) + // setup a request retriever middleware + fm := requestRetrieverMiddleware{} + + ctx := context.Background() + + // call an operation + _, err := operation(ctx, svc, &fm) + + // inspect any errors + if len(c.expectedErr) != 0 { + if err == nil { + t.Fatalf("expected error, got none") + } + if a, e := err.Error(), c.expectedErr; !strings.Contains(a, e) { + t.Fatalf("expect error code to contain %q, got %q", e, a) + } + return + } + if err != nil { + t.Fatalf("expect no error, got %v", err) + } - if e, a := c.expectedSigningName, fm.signingName; !strings.EqualFold(e, a) { - t.Fatalf("expect signing name as %s, got %s", e, a) - } - }) + // build the captured request + req := fm.request.Build(ctx) + // verify the built request is as expected + if e, a := c.expectedReqURL, req.URL.String(); e != a { + t.Fatalf("expect url %s, got %s", e, a) + } + + if e, a := c.expectedSigningRegion, fm.signingRegion; !strings.EqualFold(e, a) { + t.Fatalf("expect signing region as %s, got %s", e, a) + } + if e, a := c.expectedSigningName, fm.signingName; !strings.EqualFold(e, a) { + t.Fatalf("expect signing name as %s, got %s", e, a) } } diff --git a/service/s3control/internal/customizations/update_endpoint_test.go b/service/s3control/internal/customizations/update_endpoint_test.go index 002687eb1fe..4200004dea6 100644 --- a/service/s3control/internal/customizations/update_endpoint_test.go +++ b/service/s3control/internal/customizations/update_endpoint_test.go @@ -472,16 +472,7 @@ type requestRetrieverMiddleware struct { } func TestCustomEndpoint_SpecialOperations(t *testing.T) { - cases := map[string]struct { - options s3control.Options - operation func(context.Context, *s3control.Client, *requestRetrieverMiddleware) (interface{}, error) - expectedReqURL string - expectedSigningName string - expectedSigningRegion string - expectedHeaderForOutpostID string - expectedErr string - expectedHeaderForAccountID bool - }{ + cases := map[string]testCaseForEndpointCustomization{ "CreateBucketOperation": { options: s3control.Options{ Region: "us-west-2", @@ -579,61 +570,208 @@ func TestCustomEndpoint_SpecialOperations(t *testing.T) { for name, c := range cases { t.Run(name, func(t *testing.T) { + runValidations(t, c) + }) + } +} - // options - opts := c.options.Copy() - opts.Credentials = unit.StubCredentialsProvider{} - opts.HTTPClient = smithyhttp.NopClient{} - opts.Retryer = aws.NopRetryer{} +func runValidations(t *testing.T, c testCaseForEndpointCustomization) { + // options + opts := c.options.Copy() + opts.Credentials = unit.StubCredentialsProvider{} + opts.HTTPClient = smithyhttp.NopClient{} + opts.Retryer = aws.NopRetryer{} - // build an s3control client - svc := s3control.New(opts) - // setup a request retriever middleware - fm := requestRetrieverMiddleware{} + // build an s3control client + svc := s3control.New(opts) + // setup a request retriever middleware + fm := requestRetrieverMiddleware{} - ctx := context.Background() + ctx := context.Background() - // call an operation - _, err := c.operation(ctx, svc, &fm) + // call an operation + _, err := c.operation(ctx, svc, &fm) + + // inspect any errors + if len(c.expectedErr) != 0 { + if err == nil { + t.Fatalf("expected error, got none") + } + if a, e := err.Error(), c.expectedErr; !strings.Contains(a, e) { + t.Fatalf("expect error code to contain %q, got %q", e, a) + } + return + } + if err != nil { + t.Fatalf("expect no error, got %v", err) + } - // inspect any errors - if len(c.expectedErr) != 0 { - if err == nil { - t.Fatalf("expected error, got none") - } - if a, e := err.Error(), c.expectedErr; !strings.Contains(a, e) { - t.Fatalf("expect error code to contain %q, got %q", e, a) - } - return - } - if err != nil { - t.Fatalf("expect no error, got %v", err) - } + // build the captured request + req := fm.request.Build(ctx) + // verify the built request is as expected + if e, a := c.expectedReqURL, req.URL.String(); e != a { + t.Fatalf("expect url %s, got %s", e, a) + } - // build the captured request - req := fm.request - // verify the built request is as expected - if e, a := c.expectedReqURL, req.URL.String(); e != a { - t.Fatalf("expect url %s, got %s", e, a) - } + if e, a := c.expectedSigningRegion, fm.signingRegion; !strings.EqualFold(e, a) { + t.Fatalf("expect signing region as %s, got %s", e, a) + } - if e, a := c.expectedSigningRegion, fm.signingRegion; !strings.EqualFold(e, a) { - t.Fatalf("expect signing region as %s, got %s", e, a) - } + if e, a := c.expectedSigningName, fm.signingName; !strings.EqualFold(e, a) { + t.Fatalf("expect signing name as %s, got %s", e, a) + } - if e, a := c.expectedSigningName, fm.signingName; !strings.EqualFold(e, a) { - t.Fatalf("expect signing name as %s, got %s", e, a) - } + if c.expectedHeaderForAccountID { + if e, a := "123456789012", req.Header.Get("x-amz-account-id"); e != a { + t.Fatalf("expect account id header value to be %v, got %v", e, a) + } + } - if c.expectedHeaderForAccountID { - if e, a := "123456789012", req.Header.Get("x-amz-account-id"); e != a { - t.Fatalf("expect account id header value to be %v, got %v", e, a) - } - } + if e, a := c.expectedHeaderForOutpostID, req.Header.Get("x-amz-outpost-id"); e != a { + t.Fatalf("expect outpost id header value to be %v, got %v", e, a) + } +} - if e, a := c.expectedHeaderForOutpostID, req.Header.Get("x-amz-outpost-id"); e != a { - t.Fatalf("expect outpost id header value to be %v, got %v", e, a) - } +type testCaseForEndpointCustomization struct { + options s3control.Options + operation func(context.Context, *s3control.Client, *requestRetrieverMiddleware) (interface{}, error) + expectedReqURL string + expectedSigningName string + expectedSigningRegion string + expectedHeaderForOutpostID string + expectedErr string + expectedHeaderForAccountID bool +} + +func TestVPC_CustomEndpoint(t *testing.T) { + account := "123456789012" + cases := map[string]testCaseForEndpointCustomization{ + "standard GetAccesspoint with custom endpoint url": { + options: s3control.Options{ + EndpointResolver: s3control.EndpointResolverFromURL("https://beta.example.com"), + Region: "us-west-2", + }, + operation: func(ctx context.Context, svc *s3control.Client, fm *requestRetrieverMiddleware) (interface{}, error) { + return svc.GetAccessPoint(ctx, &s3control.GetAccessPointInput{ + AccountId: aws.String(account), + Name: aws.String("apname"), + }, addRequestRetriever(fm)) + }, + expectedReqURL: "https://123456789012.beta.example.com/v20180820/accesspoint/apname", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", + }, + "Outpost Accesspoint ARN with GetAccesspoint and custom endpoint url": { + options: s3control.Options{ + EndpointResolver: s3control.EndpointResolverFromURL( + "https://beta.example.com", + ), + Region: "us-west-2", + }, + operation: func(ctx context.Context, svc *s3control.Client, fm *requestRetrieverMiddleware) (interface{}, error) { + return svc.GetAccessPoint(ctx, &s3control.GetAccessPointInput{ + AccountId: aws.String(account), + Name: aws.String("arn:aws:s3-outposts:us-west-2:123456789012:outpost:op-01234567890123456:accesspoint:myaccesspoint"), + }, addRequestRetriever(fm)) + }, + expectedReqURL: "https://beta.example.com/v20180820/accesspoint/myaccesspoint", + expectedSigningName: "s3-outposts", + expectedSigningRegion: "us-west-2", + expectedHeaderForOutpostID: "op-01234567890123456", + }, + "standard CreateBucket with custom endpoint url": { + options: s3control.Options{ + EndpointResolver: s3control.EndpointResolverFromURL("https://beta.example.com"), + Region: "us-west-2", + }, + operation: func(ctx context.Context, svc *s3control.Client, fm *requestRetrieverMiddleware) (interface{}, error) { + return svc.CreateBucket(ctx, &s3control.CreateBucketInput{ + Bucket: aws.String("bucketname"), + OutpostId: aws.String("op-01234567890123456"), + }, addRequestRetriever(fm)) + }, + expectedReqURL: "https://beta.example.com/v20180820/bucket/bucketname", + expectedSigningName: "s3-outposts", + expectedSigningRegion: "us-west-2", + expectedHeaderForOutpostID: "op-01234567890123456", + }, + "Outpost Accesspoint for GetBucket with custom endpoint url": { + options: s3control.Options{ + EndpointResolver: s3control.EndpointResolverFromURL("https://beta.example.com"), + Region: "us-west-2", + }, + operation: func(ctx context.Context, svc *s3control.Client, fm *requestRetrieverMiddleware) (interface{}, error) { + return svc.GetBucket(ctx, &s3control.GetBucketInput{ + Bucket: aws.String("arn:aws:s3-outposts:us-west-2:123456789012:outpost:op-01234567890123456:bucket:mybucket"), + }, addRequestRetriever(fm)) + }, + expectedReqURL: "https://beta.example.com/v20180820/bucket/mybucket", + expectedSigningName: "s3-outposts", + expectedSigningRegion: "us-west-2", + expectedHeaderForOutpostID: "op-01234567890123456", + }, + "GetAccesspoint with dualstack and custom endpoint url": { + options: s3control.Options{ + EndpointResolver: s3control.EndpointResolverFromURL("https://beta.example.com"), + Region: "us-west-2", + UseDualstack: true, + }, + operation: func(ctx context.Context, svc *s3control.Client, fm *requestRetrieverMiddleware) (interface{}, error) { + return svc.GetAccessPoint(ctx, &s3control.GetAccessPointInput{ + AccountId: aws.String(account), + Name: aws.String("apname"), + }, addRequestRetriever(fm)) + }, + expectedReqURL: "https://123456789012.beta.example.com/v20180820/accesspoint/apname", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", + }, + "GetAccesspoint with Outposts accesspoint ARN and dualstack": { + options: s3control.Options{ + EndpointResolver: s3control.EndpointResolverFromURL("https://beta.example.com"), + Region: "us-west-2", + UseDualstack: true, + }, + operation: func(ctx context.Context, svc *s3control.Client, fm *requestRetrieverMiddleware) (interface{}, error) { + return svc.GetAccessPoint(ctx, &s3control.GetAccessPointInput{ + AccountId: aws.String(account), + Name: aws.String("arn:aws:s3-outposts:us-west-2:123456789012:outpost:op-01234567890123456:accesspoint:myaccesspoint"), + }, addRequestRetriever(fm)) + }, + expectedErr: "client configured for S3 Dual-stack but is not supported with resource ARN", + }, + "standard CreateBucket with dualstack": { + options: s3control.Options{ + EndpointResolver: s3control.EndpointResolverFromURL("https://beta.example.com"), + Region: "us-west-2", + UseDualstack: true, + }, + operation: func(ctx context.Context, svc *s3control.Client, fm *requestRetrieverMiddleware) (interface{}, error) { + return svc.CreateBucket(ctx, &s3control.CreateBucketInput{ + Bucket: aws.String("bucketname"), + OutpostId: aws.String("op-1234567890123456"), + }, addRequestRetriever(fm)) + }, + expectedErr: " dualstack is not supported for outposts request", + }, + "GetBucket with Outpost bucket ARN": { + options: s3control.Options{ + EndpointResolver: s3control.EndpointResolverFromURL("https://beta.example.com"), + Region: "us-west-2", + UseDualstack: true, + }, + operation: func(ctx context.Context, svc *s3control.Client, fm *requestRetrieverMiddleware) (interface{}, error) { + return svc.GetBucket(ctx, &s3control.GetBucketInput{ + Bucket: aws.String("arn:aws:s3-outposts:us-west-2:123456789012:outpost:op-01234567890123456:bucket:mybucket"), + }, addRequestRetriever(fm)) + }, + expectedErr: "client configured for S3 Dual-stack but is not supported with resource ARN", + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + runValidations(t, c) }) } } @@ -705,3 +843,15 @@ func (rm *requestRetrieverMiddleware) HandleSerialize( return next.HandleSerialize(ctx, in) } + +var addRequestRetriever = func(fm *requestRetrieverMiddleware) func(options *s3control.Options) { + return func(options *s3control.Options) { + // append request retriever middleware for request inspection + options.APIOptions = append(options.APIOptions, + func(stack *middleware.Stack) error { + // adds AFTER operation serializer middleware + stack.Serialize.Insert(fm, "OperationSerializer", middleware.After) + return nil + }) + } +} From cd74455074133282371c696f31eed147ddc7f2dc Mon Sep 17 00:00:00 2001 From: skotambkar Date: Sat, 23 Jan 2021 00:17:46 -0800 Subject: [PATCH 7/9] adds example usage for s3 vpc interface endpoints --- .../service/s3/usingPrivateLink/LICENSE.txt | 202 ++++++++++++++++++ example/service/s3/usingPrivateLink/README.md | 42 ++++ example/service/s3/usingPrivateLink/go.mod | 30 +++ .../s3/usingPrivateLink/usingPrivateLink.go | 119 +++++++++++ 4 files changed, 393 insertions(+) create mode 100644 example/service/s3/usingPrivateLink/LICENSE.txt create mode 100644 example/service/s3/usingPrivateLink/README.md create mode 100644 example/service/s3/usingPrivateLink/go.mod create mode 100644 example/service/s3/usingPrivateLink/usingPrivateLink.go diff --git a/example/service/s3/usingPrivateLink/LICENSE.txt b/example/service/s3/usingPrivateLink/LICENSE.txt new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/example/service/s3/usingPrivateLink/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/example/service/s3/usingPrivateLink/README.md b/example/service/s3/usingPrivateLink/README.md new file mode 100644 index 00000000000..f1928c490d6 --- /dev/null +++ b/example/service/s3/usingPrivateLink/README.md @@ -0,0 +1,42 @@ +# Example + +This example demonstrates how you can use the AWS SDK for Go V2's Amazon S3 client +to use AWS PrivateLink for Amazon S3. + +# Usage + +## How to build vpc endpoint url + +To access S3 bucket data using the s3 interface endpoints, prefix the vpc +endpoint with `bucket`. For eg, use endpoint url as `https://bucket.vpce-0xxxxxxx-xxx8xxg.s3.us-west-2.vpce.amazonaws.com` +to access S3 bucket data via the associated vpc endpoint. + +To access S3 access point data using the s3 interface endpoints, prefix the vpc +endpoint with `accesspoint`. For eg, use endpoint url as `https://accesspoint.vpce-0xxxxxxx-xxxx8xxg.s3.us-west-2.vpce.amazonaws.com` +to access S3 access point data via the associated vpc endpoint. + +To work with S3 control using the s3 interface endpoints, prefix the vpc endpoint +with `control`. For eg, use endpoint url as `https://control.vpce-0xxxxxxx-xxx8xxg.s3.us-west-2.vpce.amazonaws.com` +to use S3 Control operations with the associated vpc endpoint. + +## How to use the built vpc endpoint url + +Provide the vpc endpoint url (built as per instructions above) to the SDK by providing a custom endpoint resolver on +client config or as a part of request options. The VPC endpoint url should be provided as a custom endpoint, +which means the Endpoint source should be set as EndpointSourceCustom. + +You can use client's `EndpointResolverFromURL` utility which by default sets the Endpoint source as Custom source. +Note that the SDK may mutate the vpc endpoint as per the input provided to work with ARNs, unless you explicitly set +HostNameImmutable property for the endpoint. + +The example will create s3 client's that use appropriate vpc endpoint url. The example +will then create a bucket of the name provided in code. Replace the value of +the `accountID` const with the account ID for your AWS account. The +`vpcBucketEndpointUrl`, `vpcAccesspointEndpoint`, `vpcControlEndpoint`, `bucket`, +`keyName`, and `accessPoint` const variables need to be updated to match the name +of the appropriate vpc endpoint, Bucket, Object Key, and Access Point that will be +created by the example. + +```sh + AWS_REGION= go run -tags example usingPrivateLink.go + ``` diff --git a/example/service/s3/usingPrivateLink/go.mod b/example/service/s3/usingPrivateLink/go.mod new file mode 100644 index 00000000000..cf73a3b3965 --- /dev/null +++ b/example/service/s3/usingPrivateLink/go.mod @@ -0,0 +1,30 @@ +module github.com/aws/aws-sdk-go-v2/example/service/s3/usingPrivateLink + +go 1.15 + +require ( + github.com/aws/aws-sdk-go-v2 v1.0.1-0.20210122214637-6cf9ad2f8e2f + github.com/aws/aws-sdk-go-v2/config v1.0.0 + github.com/aws/aws-sdk-go-v2/service/s3 v1.0.0 + github.com/aws/aws-sdk-go-v2/service/s3control v1.0.0 +) + +replace github.com/aws/aws-sdk-go-v2/config => ../../../../config/ + +replace github.com/aws/aws-sdk-go-v2/service/s3 => ../../../../service/s3/ + +replace github.com/aws/aws-sdk-go-v2/service/s3control => ../../../../service/s3control/ + +replace github.com/aws/aws-sdk-go-v2 => ../../../../ + +replace github.com/aws/aws-sdk-go-v2/credentials => ../../../../credentials/ + +replace github.com/aws/aws-sdk-go-v2/feature/ec2/imds => ../../../../feature/ec2/imds/ + +replace github.com/aws/aws-sdk-go-v2/service/sts => ../../../../service/sts/ + +replace github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding => ../../../../service/internal/accept-encoding/ + +replace github.com/aws/aws-sdk-go-v2/service/internal/s3shared => ../../../../service/internal/s3shared/ + +replace github.com/aws/aws-sdk-go-v2/service/internal/presigned-url => ../../../../service/internal/presigned-url/ diff --git a/example/service/s3/usingPrivateLink/usingPrivateLink.go b/example/service/s3/usingPrivateLink/usingPrivateLink.go new file mode 100644 index 00000000000..3915cc9f851 --- /dev/null +++ b/example/service/s3/usingPrivateLink/usingPrivateLink.go @@ -0,0 +1,119 @@ +package main + +import ( + "context" + "flag" + "fmt" + "io/ioutil" + "log" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/arn" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3control" +) + +const ( + bucketName = "myBucketName" + accountID = "123456789012" + accessPoint = "accesspointname" + + // vpcBucketEndpoint will be used by the SDK to resolve an endpoint, when making a call to + // access `bucket` data using s3 interface endpoint. This endpoint may be mutated by the SDK, + // as per the input provided to work with ARNs. + vpcBucketEndpoint = "https://bucket.vpce-0xxxxxxx-xxx8xxg.s3.us-west-2.vpce.amazonaws.com" + + // vpcAccesspointEndpoint will be used by the SDK to resolve an endpoint, when making a call to + // access `access-point` data using s3 interface endpoint. This endpoint may be mutated by the SDK, + // as per the input provided to work with ARNs. + vpcAccesspointEndpoint = "https://accesspoint.vpce-0xxxxxxx-xxx8xxg.s3.us-west-2.vpce.amazonaws.com" + + // vpcControlEndpoint will be used by the SDK to resolve an endpoint, when making a call to + // access `control` data using s3 interface endpoint. This endpoint may be mutated by the SDK, + // as per the input provided to work with ARNs. + vpcControlEndpoint = "https://control.vpce-0xxxxxxx-xxx8xxg.s3.us-west-2.vpce.amazonaws.com" +) + +func main() { + flag.Parse() + if len(bucketName) == 0 { + flag.PrintDefaults() + log.Fatalf("invalid parameters, bucket name required") + } + + // Load the SDK's configuration from environment and shared config, and + // create the client with this. + cfg, err := config.LoadDefaultConfig(context.TODO()) + if err != nil { + log.Fatalf("failed to load SDK configuration, %v", err) + } + + s3Client := s3.NewFromConfig(cfg) + s3controlClient := s3control.NewFromConfig(cfg) + + // Create an S3 Bucket + fmt.Println("create s3 bucket") + + setVPCBucketEndpoint := s3.WithEndpointResolver(s3.EndpointResolverFromURL(vpcBucketEndpoint)) + createBucketParams := &s3.CreateBucketInput{ + Bucket: aws.String(bucketName), + } + _, err = s3Client.CreateBucket(context.TODO(), createBucketParams, setVPCBucketEndpoint) + if err != nil { + panic(fmt.Errorf("failed to create bucket: %v", err)) + } + + // Wait for S3 Bucket to Exist + fmt.Println("wait for s3 bucket to exist") + waiter := s3.NewBucketExistsWaiter(s3Client) + err = waiter.Wait(context.TODO(), &s3.HeadBucketInput{ + Bucket: aws.String(bucketName), + }, 120*time.Second) + if err != nil { + panic(fmt.Sprintf("bucket failed to materialize: %v", err)) + } + + // Create an Access Point referring to the bucket + fmt.Println("create an access point") + + setVpcControlEndpoint := s3control.WithEndpointResolver(s3control.EndpointResolverFromURL(vpcControlEndpoint)) + createAccesspointInput := &s3control.CreateAccessPointInput{ + AccountId: aws.String(accountID), + Bucket: aws.String(bucketName), + Name: aws.String(accessPoint), + } + _, err = s3controlClient.CreateAccessPoint(context.TODO(), createAccesspointInput, setVpcControlEndpoint) + if err != nil { + panic(fmt.Sprintf("failed to create access point: %v", err)) + } + + // build an arn + apARN := arn.ARN{ + Partition: "aws", + Service: "s3", + Region: cfg.Region, + AccountID: accountID, + Resource: "accesspoint/" + accessPoint, + } + + // get object using access point ARN + fmt.Println("get object using access point") + + setVPCAccesspointEndpoint := s3.WithEndpointResolver(s3.EndpointResolverFromURL(vpcAccesspointEndpoint)) + getObjectInput := &s3.GetObjectInput{ + Bucket: aws.String(apARN.String()), + Key: aws.String("somekey"), + } + + getObjectOutput, err := s3Client.GetObject(context.TODO(), getObjectInput, setVPCAccesspointEndpoint) + if err != nil { + panic(fmt.Sprintf("failed get object request: %v", err)) + } + + _, err = ioutil.ReadAll(getObjectOutput.Body) + if err != nil { + panic(fmt.Sprintf("failed to read object body: %v", err)) + } +} From 480a459d829b0383766a20270c86be0b3e83c818 Mon Sep 17 00:00:00 2001 From: skotambkar Date: Fri, 5 Feb 2021 12:18:27 -0800 Subject: [PATCH 8/9] feedback changes for example and use of aws endpoint source enums --- example/service/s3/usingPrivateLink/README.md | 2 +- example/service/s3/usingPrivateLink/go.mod | 2 +- example/service/s3/usingPrivateLink/usingPrivateLink.go | 1 - .../s3control/internal/customizations/process_arn_resource.go | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/example/service/s3/usingPrivateLink/README.md b/example/service/s3/usingPrivateLink/README.md index f1928c490d6..04d5879a2fd 100644 --- a/example/service/s3/usingPrivateLink/README.md +++ b/example/service/s3/usingPrivateLink/README.md @@ -38,5 +38,5 @@ of the appropriate vpc endpoint, Bucket, Object Key, and Access Point that will created by the example. ```sh - AWS_REGION= go run -tags example usingPrivateLink.go + AWS_REGION= go run usingPrivateLink.go ``` diff --git a/example/service/s3/usingPrivateLink/go.mod b/example/service/s3/usingPrivateLink/go.mod index cf73a3b3965..057a7c6de08 100644 --- a/example/service/s3/usingPrivateLink/go.mod +++ b/example/service/s3/usingPrivateLink/go.mod @@ -3,7 +3,7 @@ module github.com/aws/aws-sdk-go-v2/example/service/s3/usingPrivateLink go 1.15 require ( - github.com/aws/aws-sdk-go-v2 v1.0.1-0.20210122214637-6cf9ad2f8e2f + github.com/aws/aws-sdk-go-v2 v1.1.0 github.com/aws/aws-sdk-go-v2/config v1.0.0 github.com/aws/aws-sdk-go-v2/service/s3 v1.0.0 github.com/aws/aws-sdk-go-v2/service/s3control v1.0.0 diff --git a/example/service/s3/usingPrivateLink/usingPrivateLink.go b/example/service/s3/usingPrivateLink/usingPrivateLink.go index 3915cc9f851..bb1d279c015 100644 --- a/example/service/s3/usingPrivateLink/usingPrivateLink.go +++ b/example/service/s3/usingPrivateLink/usingPrivateLink.go @@ -37,7 +37,6 @@ const ( ) func main() { - flag.Parse() if len(bucketName) == 0 { flag.PrintDefaults() log.Fatalf("invalid parameters, bucket name required") diff --git a/service/s3control/internal/customizations/process_arn_resource.go b/service/s3control/internal/customizations/process_arn_resource.go index 4ef83545afd..8b19317bb59 100644 --- a/service/s3control/internal/customizations/process_arn_resource.go +++ b/service/s3control/internal/customizations/process_arn_resource.go @@ -274,7 +274,7 @@ func buildOutpostAccessPointRequest(ctx context.Context, options outpostAccessPo var endpoint aws.Endpoint var err error endpointSource := awsmiddleware.GetEndpointSource(ctx) - if endpointsID == "s3" && endpointSource != aws.EndpointSourceCustom { + if endpointsID == "s3" && endpointSource == aws.EndpointSourceServiceMetadata { // use s3 endpoint resolver endpoint, err = s3endpoints.New().ResolveEndpoint(resolveRegion, s3endpoints.Options{ DisableHTTPS: options.EndpointResolverOptions.DisableHTTPS, From 264e42ee870e1ab94d58263364f8c74f9fc98d39 Mon Sep 17 00:00:00 2001 From: skotambkar Date: Fri, 5 Feb 2021 12:22:20 -0800 Subject: [PATCH 9/9] adds change notes for aws, example, s3, s3control, internal_s3sahred modules --- ....s3.usingPrivateLink-feature-1612549025484475000.json | 9 +++++++++ .../next-release/sdk-feature-1612548857119143000.json | 9 +++++++++ ...ce.internal.s3shared-feature-1612556486110454000.json | 9 +++++++++ .../service.s3-feature-1612549083735299000.json | 9 +++++++++ .../service.s3control-feature-1612549100705399000.json | 9 +++++++++ 5 files changed, 45 insertions(+) create mode 100644 .changes/next-release/example.service.s3.usingPrivateLink-feature-1612549025484475000.json create mode 100644 .changes/next-release/sdk-feature-1612548857119143000.json create mode 100644 .changes/next-release/service.internal.s3shared-feature-1612556486110454000.json create mode 100644 .changes/next-release/service.s3-feature-1612549083735299000.json create mode 100644 .changes/next-release/service.s3control-feature-1612549100705399000.json diff --git a/.changes/next-release/example.service.s3.usingPrivateLink-feature-1612549025484475000.json b/.changes/next-release/example.service.s3.usingPrivateLink-feature-1612549025484475000.json new file mode 100644 index 00000000000..9d6b8719d1b --- /dev/null +++ b/.changes/next-release/example.service.s3.usingPrivateLink-feature-1612549025484475000.json @@ -0,0 +1,9 @@ +{ + "ID": "example.service.s3.usingPrivateLink-feature-1612549025484475000", + "SchemaVersion": 1, + "Module": "example/service/s3/usingPrivateLink", + "Type": "feature", + "Description": "adds example to use s3 vpc endpoint interface", + "MinVersion": "", + "AffectedModules": null +} \ No newline at end of file diff --git a/.changes/next-release/sdk-feature-1612548857119143000.json b/.changes/next-release/sdk-feature-1612548857119143000.json new file mode 100644 index 00000000000..004f3ee3b55 --- /dev/null +++ b/.changes/next-release/sdk-feature-1612548857119143000.json @@ -0,0 +1,9 @@ +{ + "ID": "sdk-feature-1612548857119143000", + "SchemaVersion": 1, + "Module": "/", + "Type": "feature", + "Description": "support to add endpoint source on context. Adds getter/setter for the same", + "MinVersion": "", + "AffectedModules": null +} \ No newline at end of file diff --git a/.changes/next-release/service.internal.s3shared-feature-1612556486110454000.json b/.changes/next-release/service.internal.s3shared-feature-1612556486110454000.json new file mode 100644 index 00000000000..1f3c187fd10 --- /dev/null +++ b/.changes/next-release/service.internal.s3shared-feature-1612556486110454000.json @@ -0,0 +1,9 @@ +{ + "ID": "service.internal.s3shared-feature-1612556486110454000", + "SchemaVersion": 1, + "Module": "service/internal/s3shared", + "Type": "feature", + "Description": "adds support for s3 vpc endpoint interface", + "MinVersion": "", + "AffectedModules": null +} \ No newline at end of file diff --git a/.changes/next-release/service.s3-feature-1612549083735299000.json b/.changes/next-release/service.s3-feature-1612549083735299000.json new file mode 100644 index 00000000000..067a2140113 --- /dev/null +++ b/.changes/next-release/service.s3-feature-1612549083735299000.json @@ -0,0 +1,9 @@ +{ + "ID": "service.s3-feature-1612549083735299000", + "SchemaVersion": 1, + "Module": "service/s3", + "Type": "feature", + "Description": "adds support for s3 vpc endpoint interface", + "MinVersion": "", + "AffectedModules": null +} \ No newline at end of file diff --git a/.changes/next-release/service.s3control-feature-1612549100705399000.json b/.changes/next-release/service.s3control-feature-1612549100705399000.json new file mode 100644 index 00000000000..3ae807d413d --- /dev/null +++ b/.changes/next-release/service.s3control-feature-1612549100705399000.json @@ -0,0 +1,9 @@ +{ + "ID": "service.s3control-feature-1612549100705399000", + "SchemaVersion": 1, + "Module": "service/s3control", + "Type": "feature", + "Description": "adds support for s3 vpc endpoint interface", + "MinVersion": "", + "AffectedModules": null +} \ No newline at end of file