update codegen to use accept encoding gzip customization for s3

codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/AwsCustomGoDependency.java

@@ -25,6 +25,7 @@ public final class AwsCustomGoDependency extends AwsGoDependency {
     public static final GoDependency DYNAMODB_CUSTOMIZATION = aws("service/dynamodb/internal/customizations", "ddbcust");
     public static final GoDependency S3_CUSTOMIZATION = aws("service/s3/internal/customizations", "s3cust");
     public static final GoDependency APIGATEWAY_CUSTOMIZATION = aws("service/apigateway/internal/customizations", "agcust");
+    public static final GoDependency SERVICE_INTERNAL_CUSTOMIZATION = aws("service/internal/customizations", "internalcust");
 
     private AwsCustomGoDependency() {
         super();

codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/DynamoDBValidateResponseChecksum.java

@@ -82,9 +82,9 @@ private void writeMiddlewareHelper(GoWriter writer) {
         writer.openBlock("func $L(stack *middleware.Stack, options Options) {", "}", GZIP_ADDER, () -> {
             writer.write("$T(stack, $T{Enable: options.$L})",
                     SymbolUtils.createValueSymbolBuilder(GZIP_INTERNAL_ADDER,
-                            AwsCustomGoDependency.DYNAMODB_CUSTOMIZATION).build(),
+                            AwsCustomGoDependency.SERVICE_INTERNAL_CUSTOMIZATION).build(),
                     SymbolUtils.createValueSymbolBuilder(GZIP_INTERNAL_ADDER + "Options",
-                            AwsCustomGoDependency.DYNAMODB_CUSTOMIZATION).build(),
+                            AwsCustomGoDependency.SERVICE_INTERNAL_CUSTOMIZATION).build(),
                     GZIP_CLIENT_OPTION
             );
         });

codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/S3AcceptEncodingGzip.java

@@ -0,0 +1,111 @@
+package software.amazon.smithy.aws.go.codegen.customization;
+
+import java.util.List;
+import java.util.logging.Logger;
+import software.amazon.smithy.aws.go.codegen.AddAwsConfigFields;
+import software.amazon.smithy.aws.traits.ServiceTrait;
+import software.amazon.smithy.codegen.core.SymbolProvider;
+import software.amazon.smithy.go.codegen.GoDelegator;
+import software.amazon.smithy.go.codegen.GoSettings;
+import software.amazon.smithy.go.codegen.GoWriter;
+import software.amazon.smithy.go.codegen.SymbolUtils;
+import software.amazon.smithy.go.codegen.integration.ConfigField;
+import software.amazon.smithy.go.codegen.integration.GoIntegration;
+import software.amazon.smithy.go.codegen.integration.MiddlewareRegistrar;
+import software.amazon.smithy.go.codegen.integration.RuntimeClientPlugin;
+import software.amazon.smithy.model.Model;
+import software.amazon.smithy.model.shapes.ServiceShape;
+import software.amazon.smithy.utils.ListUtils;
+
+/**
+ * S3AcceptEncodingGzip adds a customization for s3 client to disable
+ * auto decoding of GZip content by Golang HTTP Client.
+ *
+ * This customization provides an option on the S3 client options to enable
+ * AcceptEncoding for GZIP. The flag if set, will enable auto decompression of
+ * GZIP by the S3 Client.
+ *
+ * By default, the client's auto decompression of GZIP content is turned off.
+ */
+public class S3AcceptEncodingGzip implements GoIntegration {
+    private static final Logger LOGGER = Logger.getLogger(AddAwsConfigFields.class.getName());
+
+    private static final String GZIP_CLIENT_OPTION = "EnableAcceptEncodingGzip";
+    private static final String GZIP_ADDER = "addAcceptEncodingGzip";
+    private static final String GZIP_INTERNAL_ADDER = "AddAcceptEncodingGzip";
+
+    /**
+     * Gets the sort order of the customization from -128 to 127, with lowest
+     * executed first.
+     *
+     * @return Returns the sort order, defaults to -50.
+     */
+    @Override
+    public byte getOrder() {
+        return 127;
+    }
+
+    @Override
+    public void writeAdditionalFiles(
+            GoSettings settings,
+            Model model,
+            SymbolProvider symbolProvider,
+            GoDelegator goDelegator
+    ) {
+        if (!isServiceS3(model, settings.getService(model))) {
+            return;
+        }
+
+        goDelegator.useShapeWriter(settings.getService(model), this::writeMiddlewareHelper);
+    }
+
+    private void writeMiddlewareHelper(GoWriter writer) {
+        writer.openBlock("func $L(stack *middleware.Stack, options Options) {", "}", GZIP_ADDER, () -> {
+            writer.write("$T(stack, $T{Enable: options.$L})",
+                    SymbolUtils.createValueSymbolBuilder(GZIP_INTERNAL_ADDER,
+                            AwsCustomGoDependency.SERVICE_INTERNAL_CUSTOMIZATION).build(),
+                    SymbolUtils.createValueSymbolBuilder(GZIP_INTERNAL_ADDER + "Options",
+                            AwsCustomGoDependency.SERVICE_INTERNAL_CUSTOMIZATION).build(),
+                    GZIP_CLIENT_OPTION
+            );
+        });
+        writer.insertTrailingNewline();
+    }
+
+    @Override
+    public List<RuntimeClientPlugin> getClientPlugins() {
+        return ListUtils.of(
+                // Add s3 explicit control over accept-encoding: gzip.
+                RuntimeClientPlugin.builder()
+                        .servicePredicate(S3AcceptEncodingGzip::isServiceS3)
+                        .configFields(ListUtils.of(
+                                ConfigField.builder()
+                                        .name(GZIP_CLIENT_OPTION)
+                                        .type(SymbolUtils.createValueSymbolBuilder("bool")
+                                                .putProperty(SymbolUtils.GO_UNIVERSE_TYPE, true)
+                                                .build())
+                                        .documentation("Allows you to enable the client's support for "
+                                                + "compressed gzip responses. Disabled by default.")
+                                        .build()
+                        ))
+                        .registerMiddleware(MiddlewareRegistrar.builder()
+                                .resolvedFunction(SymbolUtils.createValueSymbolBuilder(GZIP_ADDER)
+                                        .build())
+                                .useClientOptions()
+                                .build()
+                        )
+                        .build()
+        );
+    }
+
+    /**
+     * Return true if service is S3.
+     *
+     * @param model the model used for generation.
+     * @param service the service shape for which default HTTP Client is generated.
+     * @return true if service is S3
+     */
+    private static boolean isServiceS3(Model model, ServiceShape service) {
+        return service.expectTrait(ServiceTrait.class).getSdkId().equalsIgnoreCase("S3");
+    }
+}

codegen/smithy-aws-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration

@@ -11,3 +11,4 @@ software.amazon.smithy.aws.go.codegen.customization.DynamoDBValidateResponseChec
 software.amazon.smithy.aws.go.codegen.customization.S3UpdateEndpoint
 software.amazon.smithy.aws.go.codegen.customization.APIGatewayAcceptHeader
 software.amazon.smithy.aws.go.codegen.customization.BackfillOptionalAuthTrait
+software.amazon.smithy.aws.go.codegen.customization.S3AcceptEncodingGzip

service/internal/customizations/accept_encoding_gzip.go

@@ -0,0 +1,168 @@
+package customizations
+
+import (
+	"compress/gzip"
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/awslabs/smithy-go"
+	"github.com/awslabs/smithy-go/middleware"
+	smithyhttp "github.com/awslabs/smithy-go/transport/http"
+)
+
+const acceptEncodingHeaderKey = "Accept-Encoding"
+const contentEncodingHeaderKey = "Content-Encoding"
+
+// AddAcceptEncodingGzipOptions provides the options for the
+// AddAcceptEncodingGzip middleware setup.
+type AddAcceptEncodingGzipOptions struct {
+	Enable bool
+}
+
+// AddAcceptEncodingGzip explicitly adds handling for accept-encoding GZIP
+// middleware to the operation stack. This allows checksums to be correctly
+// computed without disabling GZIP support.
+func AddAcceptEncodingGzip(stack *middleware.Stack, options AddAcceptEncodingGzipOptions) {
+	if options.Enable {
+		stack.Finalize.Add(&AcceptEncodingGzipMiddleware{}, middleware.Before)
+		stack.Deserialize.Insert(&DecompressGzipMiddleware{}, "OperationDeserializer", middleware.After)
+		return
+	}
+
+	stack.Finalize.Add(&DisableAcceptEncodingGzipMiddleware{}, middleware.Before)
+}
+
+// DisableAcceptEncodingGzipMiddleware provides the middleware that will
+// disable the underlying http client automatically enabling for gzip
+// decompress content-encoding support.
+type DisableAcceptEncodingGzipMiddleware struct{}
+
+// ID returns the id for the middleware.
+func (*DisableAcceptEncodingGzipMiddleware) ID() string {
+	return "DisableAcceptEncodingGzipMiddleware"
+}
+
+// HandleFinalize implements the FinalizeMiddlware interface.
+func (*DisableAcceptEncodingGzipMiddleware) HandleFinalize(
+	ctx context.Context, input middleware.FinalizeInput, next middleware.FinalizeHandler,
+) (
+	output middleware.FinalizeOutput, metadata middleware.Metadata, err error,
+) {
+	req, ok := input.Request.(*smithyhttp.Request)
+	if !ok {
+		return output, metadata, &smithy.SerializationError{
+			Err: fmt.Errorf("unknown request type %T", input.Request),
+		}
+	}
+
+	// Explicitly enable gzip support, this will prevent the http client from
+	// auto extracting the zipped content.
+	req.Header.Set(acceptEncodingHeaderKey, "identity")
+
+	return next.HandleFinalize(ctx, input)
+}
+
+// AcceptEncodingGzipMiddleware provides a middleware to enable support for
+// gzip responses, with manual decompression. This prevents the underlying HTTP
+// client from performing the gzip decompression automatically.
+type AcceptEncodingGzipMiddleware struct{}
+
+// ID returns the id for the middleware.
+func (*AcceptEncodingGzipMiddleware) ID() string { return "AcceptEncodingGzipMiddleware" }
+
+// HandleFinalize implements the FinalizeMiddlware interface.
+func (*AcceptEncodingGzipMiddleware) HandleFinalize(
+	ctx context.Context, input middleware.FinalizeInput, next middleware.FinalizeHandler,
+) (
+	output middleware.FinalizeOutput, metadata middleware.Metadata, err error,
+) {
+	req, ok := input.Request.(*smithyhttp.Request)
+	if !ok {
+		return output, metadata, &smithy.SerializationError{
+			Err: fmt.Errorf("unknown request type %T", input.Request),
+		}
+	}
+
+	// Explicitly enable gzip support, this will prevent the http client from
+	// auto extracting the zipped content.
+	req.Header.Set(acceptEncodingHeaderKey, "gzip")
+
+	return next.HandleFinalize(ctx, input)
+}
+
+// DecompressGzipMiddleware provides the middleware for decompressing a gzip
+// response from the service.
+type DecompressGzipMiddleware struct{}
+
+// ID returns the id for the middleware.
+func (*DecompressGzipMiddleware) ID() string { return "DecompressGzipMiddleware" }
+
+// HandleDeserialize implements the DeserializeMiddlware interface.
+func (*DecompressGzipMiddleware) HandleDeserialize(
+	ctx context.Context, input middleware.DeserializeInput, next middleware.DeserializeHandler,
+) (
+	output middleware.DeserializeOutput, metadata middleware.Metadata, err error,
+) {
+	output, metadata, err = next.HandleDeserialize(ctx, input)
+	if err != nil {
+		return output, metadata, err
+	}
+
+	resp, ok := output.RawResponse.(*smithyhttp.Response)
+	if !ok {
+		return output, metadata, &smithy.DeserializationError{
+			Err: fmt.Errorf("unknown response type %T", output.RawResponse),
+		}
+	}
+	if v := resp.Header.Get(contentEncodingHeaderKey); v != "gzip" {
+		return output, metadata, err
+	}
+
+	// Clear content length since it will no longer be valid once the response
+	// body is decompressed.
+	resp.Header.Del("Content-Length")
+	resp.ContentLength = -1
+
+	resp.Body = wrapGzipReader(resp.Body)
+
+	return output, metadata, err
+}
+
+type gzipReader struct {
+	reader io.ReadCloser
+	gzip   *gzip.Reader
+}
+
+func wrapGzipReader(reader io.ReadCloser) *gzipReader {
+	return &gzipReader{
+		reader: reader,
+	}
+}
+
+// Read wraps the gzip reader around the underlying io.Reader to extract the
+// response bytes on the fly.
+func (g *gzipReader) Read(b []byte) (n int, err error) {
+	if g.gzip == nil {
+		g.gzip, err = gzip.NewReader(g.reader)
+		if err != nil {
+			g.gzip = nil // ensure uninitialized gzip value isn't used in close.
+			return 0, fmt.Errorf("failed to decompress gzip response, %w", err)
+		}
+	}
+
+	return g.gzip.Read(b)
+}
+
+func (g *gzipReader) Close() error {
+	if g.gzip == nil {
+		return nil
+	}
+
+	if err := g.gzip.Close(); err != nil {
+		g.reader.Close()
+		return fmt.Errorf("failed to decompress gzip response, %w", err)
+	}
+
+	return g.reader.Close()
+}