From 6b963decdc6c02c2e802a817ebdb48da90b44a83 Mon Sep 17 00:00:00 2001 From: skotambkar Date: Thu, 24 Sep 2020 00:25:32 -0700 Subject: [PATCH] add customization to disable auto decode gzip for s3. Moves customizations from dynamodb to reuse middleware --- .../customization/AwsCustomGoDependency.java | 3 + .../DynamoDBValidateResponseChecksum.java | 4 +- .../customization/S3AcceptEncodingGzip.java | 98 +++++++++++++++++++ ...mithy.go.codegen.integration.GoIntegration | 3 +- .../dynamodb/internal/customizations/doc.go | 86 ++++++++-------- .../accept-encoding}/accept_encoding_gzip.go | 30 +++--- .../accept_encoding_gzip_test.go | 25 ++++- service/internal/accept-encoding/doc.go | 23 +++++ 8 files changed, 204 insertions(+), 68 deletions(-) create mode 100644 codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/S3AcceptEncodingGzip.java rename service/{dynamodb/internal/customizations => internal/accept-encoding}/accept_encoding_gzip.go (82%) rename service/{dynamodb/internal/customizations => internal/accept-encoding}/accept_encoding_gzip_test.go (92%) create mode 100644 service/internal/accept-encoding/doc.go diff --git a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/AwsCustomGoDependency.java b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/AwsCustomGoDependency.java index c79abb7ba88..28198d55c75 100644 --- a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/AwsCustomGoDependency.java +++ b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/AwsCustomGoDependency.java @@ -29,8 +29,11 @@ public final class AwsCustomGoDependency extends AwsGoDependency { "service/apigateway/internal/customizations", "agcust"); public static final GoDependency GLACIER_CUSTOMIZATION = aws( "service/glacier/internal/customizations", "glaciercust"); + public static final GoDependency S3_SHARED_CUSTOMIZATION = awsModuleDep( "service/internal/s3shared", null, "s3shared"); + public static final GoDependency ACCEPT_ENCODING_CUSTOMIZATION = awsModuleDep( + "service/internal/accept-encoding", null, "acceptencodingcust"); private AwsCustomGoDependency() { super(); diff --git a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/DynamoDBValidateResponseChecksum.java b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/DynamoDBValidateResponseChecksum.java index 4720a449b55..32f3b2caacc 100644 --- a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/DynamoDBValidateResponseChecksum.java +++ b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/DynamoDBValidateResponseChecksum.java @@ -82,9 +82,9 @@ private void writeMiddlewareHelper(GoWriter writer) { writer.openBlock("func $L(stack *middleware.Stack, options Options) {", "}", GZIP_ADDER, () -> { writer.write("$T(stack, $T{Enable: options.$L})", SymbolUtils.createValueSymbolBuilder(GZIP_INTERNAL_ADDER, - AwsCustomGoDependency.DYNAMODB_CUSTOMIZATION).build(), + AwsCustomGoDependency.ACCEPT_ENCODING_CUSTOMIZATION).build(), SymbolUtils.createValueSymbolBuilder(GZIP_INTERNAL_ADDER + "Options", - AwsCustomGoDependency.DYNAMODB_CUSTOMIZATION).build(), + AwsCustomGoDependency.ACCEPT_ENCODING_CUSTOMIZATION).build(), GZIP_CLIENT_OPTION ); }); diff --git a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/S3AcceptEncodingGzip.java b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/S3AcceptEncodingGzip.java new file mode 100644 index 00000000000..a792e651c00 --- /dev/null +++ b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/S3AcceptEncodingGzip.java @@ -0,0 +1,98 @@ +package software.amazon.smithy.aws.go.codegen.customization; + +import java.util.List; +import java.util.logging.Logger; +import software.amazon.smithy.aws.go.codegen.AddAwsConfigFields; +import software.amazon.smithy.aws.traits.ServiceTrait; +import software.amazon.smithy.codegen.core.SymbolProvider; +import software.amazon.smithy.go.codegen.GoDelegator; +import software.amazon.smithy.go.codegen.GoSettings; +import software.amazon.smithy.go.codegen.GoWriter; +import software.amazon.smithy.go.codegen.SymbolUtils; +import software.amazon.smithy.go.codegen.integration.ConfigField; +import software.amazon.smithy.go.codegen.integration.GoIntegration; +import software.amazon.smithy.go.codegen.integration.MiddlewareRegistrar; +import software.amazon.smithy.go.codegen.integration.RuntimeClientPlugin; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.utils.ListUtils; + +/** + * S3AcceptEncodingGzip adds a customization for s3 client to disable + * auto decoding of GZip content by Golang HTTP Client. + * + * This customization provides an option on the S3 client options to enable + * AcceptEncoding for GZIP. The flag if set, will enable auto decompression of + * GZIP by the S3 Client. + * + * By default, the client's auto decompression of GZIP content is turned off. + */ +public class S3AcceptEncodingGzip implements GoIntegration { + private static final Logger LOGGER = Logger.getLogger(AddAwsConfigFields.class.getName()); + + private static final String GZIP_DISABLE = "disableAcceptEncodingGzip"; + private static final String GZIP_INTERNAL_ADDER = "AddAcceptEncodingGzip"; + + /** + * Gets the sort order of the customization from -128 to 127, with lowest + * executed first. + * + * @return Returns the sort order, defaults to -50. + */ + @Override + public byte getOrder() { + return 127; + } + + @Override + public void writeAdditionalFiles( + GoSettings settings, + Model model, + SymbolProvider symbolProvider, + GoDelegator goDelegator + ) { + if (!isServiceS3(model, settings.getService(model))) { + return; + } + + goDelegator.useShapeWriter(settings.getService(model), this::writeMiddlewareHelper); + } + + private void writeMiddlewareHelper(GoWriter writer) { + writer.openBlock("func $L(stack *middleware.Stack) {", "}", GZIP_DISABLE, () -> { + writer.write("$T(stack, $T{})", + SymbolUtils.createValueSymbolBuilder(GZIP_INTERNAL_ADDER, + AwsCustomGoDependency.ACCEPT_ENCODING_CUSTOMIZATION).build(), + SymbolUtils.createValueSymbolBuilder(GZIP_INTERNAL_ADDER + "Options", + AwsCustomGoDependency.ACCEPT_ENCODING_CUSTOMIZATION).build() + ); + }); + writer.insertTrailingNewline(); + } + + @Override + public List getClientPlugins() { + return ListUtils.of( + // register disableAcceptEncodingGzip middleware + RuntimeClientPlugin.builder() + .servicePredicate(S3AcceptEncodingGzip::isServiceS3) + .registerMiddleware(MiddlewareRegistrar.builder() + .resolvedFunction(SymbolUtils.createValueSymbolBuilder(GZIP_DISABLE) + .build()) + .build() + ) + .build() + ); + } + + /** + * Return true if service is S3. + * + * @param model the model used for generation. + * @param service the service shape for which default HTTP Client is generated. + * @return true if service is S3 + */ + private static boolean isServiceS3(Model model, ServiceShape service) { + return service.expectTrait(ServiceTrait.class).getSdkId().equalsIgnoreCase("S3"); + } +} diff --git a/codegen/smithy-aws-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration b/codegen/smithy-aws-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration index d843b0d4ce5..fc4380e302d 100644 --- a/codegen/smithy-aws-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration +++ b/codegen/smithy-aws-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration @@ -17,4 +17,5 @@ software.amazon.smithy.aws.go.codegen.customization.GlacierCustomizations software.amazon.smithy.aws.go.codegen.customization.S3ResponseErrorWrapper software.amazon.smithy.aws.go.codegen.customization.S3MetadataRetriever software.amazon.smithy.aws.go.codegen.customization.S3ContentSHA256Header -software.amazon.smithy.aws.go.codegen.customization.BackfillS3ObjectSizeMemberShapeType \ No newline at end of file +software.amazon.smithy.aws.go.codegen.customization.BackfillS3ObjectSizeMemberShapeType +software.amazon.smithy.aws.go.codegen.customization.S3AcceptEncodingGzip diff --git a/service/dynamodb/internal/customizations/doc.go b/service/dynamodb/internal/customizations/doc.go index 891c87e8513..765e424b9b7 100644 --- a/service/dynamodb/internal/customizations/doc.go +++ b/service/dynamodb/internal/customizations/doc.go @@ -1,47 +1,41 @@ -// Package customizations provides customizations for the Amazon DynamoDB API client. -// -// The DynamoDB API client uses two customizations, response checksum validation, -// and manual content-encoding: gzip support. -// -// Middleware layering -// -// Checksum validation needs to be performed first in deserialization chain -// on top of gzip decompression. Since the behavior of Deserialization is -// in reverse order to the other stack steps its easier to consider that -// "after" means "before". -// -// HTTP Response -> Checksum -> gzip decompress -> deserialize -// -// Response checksum validation -// -// DynamoDB responses can include a X-Amz-Crc32 header with the CRC32 checksum -// value of the response body. If the response body is content-encoding: gzip, the -// checksum is of the gzipped response content. -// -// If the header is present, the SDK should validate that the response payload -// computed CRC32 checksum matches the value provided in the header. The checksum -// header is based on the original payload provided returned by the service. Which -// means that if the response is gzipped the checksum is of the gzipped response, -// not the decompressed response bytes. -// -// Customization option: -// DisableValidateResponseChecksum (Enabled by Default) -// -// Accept encoding gzip -// -// The Go HTTP client automatically supports accept-encoding and content-encoding -// gzip by default. This default behavior is not desired by the SDK, and prevents -// validating the response body's checksum. To prevent this the SDK must manually -// control usage of content-encoding gzip. -// -// To control content-encoding, the SDK must always set the `Accept-Encoding` -// header to a value. This prevents the HTTP client from using gzip automatically. -// When gzip is enabled on the API client, the SDK's customization will control -// decompressing the gzip data in order to not break the checksum validation. When -// gzip is disabled, the API client will disable gzip, preventing the HTTP -// client's default behavior. -// -// Customization option: -// EnableAcceptEncodingGzip (Disabled by Default) -// +/* +Package customizations provides customizations for the Amazon DynamoDB API client. + +The DynamoDB API client uses two customizations, response checksum validation, +and manual content-encoding: gzip support. + +Middleware layering + +Checksum validation needs to be performed first in deserialization chain +on top of gzip decompression. Since the behavior of Deserialization is +in reverse order to the other stack steps its easier to consider that +"after" means "before". + + HTTP Response -> Checksum -> gzip decompress -> deserialize + +Response checksum validation + +DynamoDB responses can include a X-Amz-Crc32 header with the CRC32 checksum +value of the response body. If the response body is content-encoding: gzip, the +checksum is of the gzipped response content. + +If the header is present, the SDK should validate that the response payload +computed CRC32 checksum matches the value provided in the header. The checksum +header is based on the original payload provided returned by the service. Which +means that if the response is gzipped the checksum is of the gzipped response, +not the decompressed response bytes. + +Customization option: + DisableValidateResponseChecksum (Enabled by Default) + +Accept encoding gzip + +For customization around accept encoding, dynamodb client uses the middlewares +defined at service/internal/accept-encoding. Please refer to the documentation for +`accept-encoding` package for more details. + +Customization option: + EnableAcceptEncodingGzip (Disabled by Default) + +*/ package customizations diff --git a/service/dynamodb/internal/customizations/accept_encoding_gzip.go b/service/internal/accept-encoding/accept_encoding_gzip.go similarity index 82% rename from service/dynamodb/internal/customizations/accept_encoding_gzip.go rename to service/internal/accept-encoding/accept_encoding_gzip.go index 760bfe42e36..91b30dfdcbb 100644 --- a/service/dynamodb/internal/customizations/accept_encoding_gzip.go +++ b/service/internal/accept-encoding/accept_encoding_gzip.go @@ -1,4 +1,4 @@ -package customizations +package acceptencoding import ( "compress/gzip" @@ -6,7 +6,7 @@ import ( "fmt" "io" - smithy "github.com/awslabs/smithy-go" + "github.com/awslabs/smithy-go" "github.com/awslabs/smithy-go/middleware" smithyhttp "github.com/awslabs/smithy-go/transport/http" ) @@ -25,26 +25,26 @@ type AddAcceptEncodingGzipOptions struct { // computed without disabling GZIP support. func AddAcceptEncodingGzip(stack *middleware.Stack, options AddAcceptEncodingGzipOptions) { if options.Enable { - stack.Finalize.Add(&AcceptEncodingGzipMiddleware{}, middleware.Before) + stack.Finalize.Add(&EnableGzipMiddleware{}, middleware.Before) stack.Deserialize.Insert(&DecompressGzipMiddleware{}, "OperationDeserializer", middleware.After) return } - stack.Finalize.Add(&DisableAcceptEncodingGzipMiddleware{}, middleware.Before) + stack.Finalize.Add(&DisableGzipMiddleware{}, middleware.Before) } -// DisableAcceptEncodingGzipMiddleware provides the middleware that will +// DisableGzipMiddleware provides the middleware that will // disable the underlying http client automatically enabling for gzip // decompress content-encoding support. -type DisableAcceptEncodingGzipMiddleware struct{} +type DisableGzipMiddleware struct{} // ID returns the id for the middleware. -func (*DisableAcceptEncodingGzipMiddleware) ID() string { - return "DynamoDB:DisableAcceptEncodingGzipMiddleware" +func (*DisableGzipMiddleware) ID() string { + return "DisableAcceptEncodingGzipMiddleware" } -// HandleFinalize implements the FinalizeMiddlware interface. -func (*DisableAcceptEncodingGzipMiddleware) HandleFinalize( +// HandleFinalize implements the FinalizeMiddleware interface. +func (*DisableGzipMiddleware) HandleFinalize( ctx context.Context, input middleware.FinalizeInput, next middleware.FinalizeHandler, ) ( output middleware.FinalizeOutput, metadata middleware.Metadata, err error, @@ -63,16 +63,16 @@ func (*DisableAcceptEncodingGzipMiddleware) HandleFinalize( return next.HandleFinalize(ctx, input) } -// AcceptEncodingGzipMiddleware provides a middleware to enable support for +// EnableGzipMiddleware provides a middleware to enable support for // gzip responses, with manual decompression. This prevents the underlying HTTP // client from performing the gzip decompression automatically. -type AcceptEncodingGzipMiddleware struct{} +type EnableGzipMiddleware struct{} // ID returns the id for the middleware. -func (*AcceptEncodingGzipMiddleware) ID() string { return "DynamoDB:AcceptEncodingGzipMiddleware" } +func (*EnableGzipMiddleware) ID() string { return "AcceptEncodingGzipMiddleware" } // HandleFinalize implements the FinalizeMiddlware interface. -func (*AcceptEncodingGzipMiddleware) HandleFinalize( +func (*EnableGzipMiddleware) HandleFinalize( ctx context.Context, input middleware.FinalizeInput, next middleware.FinalizeHandler, ) ( output middleware.FinalizeOutput, metadata middleware.Metadata, err error, @@ -96,7 +96,7 @@ func (*AcceptEncodingGzipMiddleware) HandleFinalize( type DecompressGzipMiddleware struct{} // ID returns the id for the middleware. -func (*DecompressGzipMiddleware) ID() string { return "DynamoDB:DecompressGzipMiddleware" } +func (*DecompressGzipMiddleware) ID() string { return "DecompressGzipMiddleware" } // HandleDeserialize implements the DeserializeMiddlware interface. func (*DecompressGzipMiddleware) HandleDeserialize( diff --git a/service/dynamodb/internal/customizations/accept_encoding_gzip_test.go b/service/internal/accept-encoding/accept_encoding_gzip_test.go similarity index 92% rename from service/dynamodb/internal/customizations/accept_encoding_gzip_test.go rename to service/internal/accept-encoding/accept_encoding_gzip_test.go index acc0e274fe6..ce8d4c16ac8 100644 --- a/service/dynamodb/internal/customizations/accept_encoding_gzip_test.go +++ b/service/internal/accept-encoding/accept_encoding_gzip_test.go @@ -1,4 +1,4 @@ -package customizations +package acceptencoding import ( "bytes" @@ -42,7 +42,7 @@ func TestAddAcceptEncodingGzip(t *testing.T) { } if c.Enable { - id = (*AcceptEncodingGzipMiddleware)(nil).ID() + id = (*EnableGzipMiddleware)(nil).ID() if m, ok := stack.Finalize.Get(id); !ok || m == nil { t.Fatalf("expect %s to be present.", id) } @@ -53,7 +53,7 @@ func TestAddAcceptEncodingGzip(t *testing.T) { } return } - id = (*AcceptEncodingGzipMiddleware)(nil).ID() + id = (*EnableGzipMiddleware)(nil).ID() if m, ok := stack.Finalize.Get(id); ok || m != nil { t.Fatalf("expect %s not to be present.", id) } @@ -67,7 +67,7 @@ func TestAddAcceptEncodingGzip(t *testing.T) { } func TestAcceptEncodingGzipMiddleware(t *testing.T) { - m := &AcceptEncodingGzipMiddleware{} + m := &EnableGzipMiddleware{} _, _, err := m.HandleFinalize(context.Background(), middleware.FinalizeInput{ @@ -196,3 +196,20 @@ func (*stubOpDeserializer) HandleDeserialize( ) { return next.HandleDeserialize(ctx, input) } + +type wasClosedReadCloser struct { + io.Reader + closed bool +} + +func (c *wasClosedReadCloser) WasClosed() bool { + return c.closed +} + +func (c *wasClosedReadCloser) Close() error { + c.closed = true + if v, ok := c.Reader.(io.Closer); ok { + return v.Close() + } + return nil +} diff --git a/service/internal/accept-encoding/doc.go b/service/internal/accept-encoding/doc.go new file mode 100644 index 00000000000..e68fee17b35 --- /dev/null +++ b/service/internal/accept-encoding/doc.go @@ -0,0 +1,23 @@ +/* +Package customizations provides customizations associated with Accept Encoding Header. + +Accept encoding gzip + +The Go HTTP client automatically supports accept-encoding and content-encoding +gzip by default. This default behavior is not desired by the SDK, and prevents +validating the response body's checksum. To prevent this the SDK must manually +control usage of content-encoding gzip. + +To control content-encoding, the SDK must always set the `Accept-Encoding` +header to a value. This prevents the HTTP client from using gzip automatically. +When gzip is enabled on the API client, the SDK's customization will control +decompressing the gzip data in order to not break the checksum validation. When +gzip is disabled, the API client will disable gzip, preventing the HTTP +client's default behavior. + +An `EnableAcceptEncodingGzip` option may or may not be present depending on the client using +the below middleware. The option if present can be used to enable auto decompressing +gzip by the SDK. + +*/ +package acceptencoding