From 8e86e9e1ec955a02e49fed67f42876ead41a1039 Mon Sep 17 00:00:00 2001 From: George Fu Date: Fri, 2 Aug 2024 17:39:05 +0000 Subject: [PATCH] chore(client-s3): compatibility for S3Express and httpsigning midware --- clients/client-s3/src/S3Client.ts | 38 +++--- clients/client-s3/test/unit/S3.spec.ts | 9 +- .../aws/typescript/codegen/AddS3Config.java | 7 + .../aws_sdk/AwsSdkSigV4Signer.ts | 19 ++- .../httpAuthSchemes/aws_sdk/index.ts | 2 +- packages/middleware-sdk-s3/package.json | 3 + .../s3ExpressHttpSigningMiddleware.ts | 124 ++++++++++++++++++ .../src/s3-express/functions/signS3Express.ts | 29 ++++ .../middleware-sdk-s3/src/s3-express/index.ts | 5 + 9 files changed, 215 insertions(+), 21 deletions(-) create mode 100644 packages/middleware-sdk-s3/src/s3-express/functions/s3ExpressHttpSigningMiddleware.ts create mode 100644 packages/middleware-sdk-s3/src/s3-express/functions/signS3Express.ts diff --git a/clients/client-s3/src/S3Client.ts b/clients/client-s3/src/S3Client.ts index 146f1067369e..40cfea69a470 100644 --- a/clients/client-s3/src/S3Client.ts +++ b/clients/client-s3/src/S3Client.ts @@ -10,6 +10,7 @@ import { getLoggerPlugin } from "@aws-sdk/middleware-logger"; import { getRecursionDetectionPlugin } from "@aws-sdk/middleware-recursion-detection"; import { getRegionRedirectMiddlewarePlugin, + getS3ExpressHttpSigningPlugin, getS3ExpressPlugin, getValidateBucketNamePlugin, resolveS3Config, @@ -674,15 +675,6 @@ export interface ClientDefaults extends Partial<__SmithyConfiguration<__HttpHand */ credentialDefaultProvider?: (input: any) => AwsCredentialIdentityProvider; - /** - * Whether to escape request path when signing the request. - */ - signingEscapePath?: boolean; - - /** - * Whether to override the request region with the region inferred from requested resource's ARN. Defaults to false. - */ - useArnRegion?: boolean | Provider; /** * Value for how many times a request will be made at most in case of retry. */ @@ -715,6 +707,15 @@ export interface ClientDefaults extends Partial<__SmithyConfiguration<__HttpHand */ defaultsMode?: __DefaultsMode | __Provider<__DefaultsMode>; + /** + * Whether to escape request path when signing the request. + */ + signingEscapePath?: boolean; + + /** + * Whether to override the request region with the region inferred from requested resource's ARN. Defaults to false. + */ + useArnRegion?: boolean | Provider; /** * The internal function that inject utilities to runtime-specific stream to help users consume the data * @internal @@ -732,9 +733,9 @@ export type S3ClientConfigType = Partial<__SmithyConfiguration<__HttpHandlerOpti RegionInputConfig & HostHeaderInputConfig & EndpointInputConfig & - S3InputConfig & EventStreamSerdeInputConfig & HttpAuthSchemeInputConfig & + S3InputConfig & ClientInputEndpointParameters; /** * @public @@ -754,9 +755,9 @@ export type S3ClientResolvedConfigType = __SmithyResolvedConfiguration<__HttpHan RegionResolvedConfig & HostHeaderResolvedConfig & EndpointResolvedConfig & - S3ResolvedConfig & EventStreamSerdeResolvedConfig & HttpAuthSchemeResolvedConfig & + S3ResolvedConfig & ClientResolvedEndpointParameters; /** * @public @@ -788,9 +789,9 @@ export class S3Client extends __Client< const _config_4 = resolveRegionConfig(_config_3); const _config_5 = resolveHostHeaderConfig(_config_4); const _config_6 = resolveEndpointConfig(_config_5); - const _config_7 = resolveS3Config(_config_6, { session: [() => this, CreateSessionCommand] }); - const _config_8 = resolveEventStreamSerdeConfig(_config_7); - const _config_9 = resolveHttpAuthSchemeConfig(_config_8); + const _config_7 = resolveEventStreamSerdeConfig(_config_6); + const _config_8 = resolveHttpAuthSchemeConfig(_config_7); + const _config_9 = resolveS3Config(_config_8, { session: [() => this, CreateSessionCommand] }); const _config_10 = resolveRuntimeExtensions(_config_9, configuration?.extensions || []); super(_config_10); this.config = _config_10; @@ -800,10 +801,6 @@ export class S3Client extends __Client< this.middlewareStack.use(getHostHeaderPlugin(this.config)); this.middlewareStack.use(getLoggerPlugin(this.config)); this.middlewareStack.use(getRecursionDetectionPlugin(this.config)); - this.middlewareStack.use(getValidateBucketNamePlugin(this.config)); - this.middlewareStack.use(getAddExpectContinuePlugin(this.config)); - this.middlewareStack.use(getRegionRedirectMiddlewarePlugin(this.config)); - this.middlewareStack.use(getS3ExpressPlugin(this.config)); this.middlewareStack.use( getHttpAuthSchemeEndpointRuleSetPlugin(this.config, { httpAuthSchemeParametersProvider: defaultS3HttpAuthSchemeParametersProvider, @@ -815,6 +812,11 @@ export class S3Client extends __Client< }) ); this.middlewareStack.use(getHttpSigningPlugin(this.config)); + this.middlewareStack.use(getValidateBucketNamePlugin(this.config)); + this.middlewareStack.use(getAddExpectContinuePlugin(this.config)); + this.middlewareStack.use(getRegionRedirectMiddlewarePlugin(this.config)); + this.middlewareStack.use(getS3ExpressPlugin(this.config)); + this.middlewareStack.use(getS3ExpressHttpSigningPlugin(this.config)); } /** diff --git a/clients/client-s3/test/unit/S3.spec.ts b/clients/client-s3/test/unit/S3.spec.ts index 34bc56d2ecc5..70bb865f208f 100644 --- a/clients/client-s3/test/unit/S3.spec.ts +++ b/clients/client-s3/test/unit/S3.spec.ts @@ -86,8 +86,9 @@ describe("Endpoints from ARN", () => { const OutpostId = "op-01234567890123456"; const AccountId = "123456789012"; const region = "us-west-2"; + const clientRegion = "us-east-1"; const credentials = { accessKeyId: "key", secretAccessKey: "secret" }; - const client = new S3({ region: "us-east-1", credentials, useArnRegion: true }); + const client = new S3({ region: clientRegion, credentials, useArnRegion: true }); client.middlewareStack.add(endpointValidator, { step: "finalizeRequest", priority: "low" }); const result: any = await client.putObject({ Bucket: `arn:aws:s3-outposts:${region}:${AccountId}:outpost/${OutpostId}/accesspoint/abc-111`, @@ -96,6 +97,12 @@ describe("Endpoints from ARN", () => { }); expect(result.request.hostname).to.eql(`abc-111-${AccountId}.${OutpostId}.s3-outposts.us-west-2.amazonaws.com`); const date = new Date().toISOString().slice(0, 10).replace(/-/g, ""); //20201029 + + /* + * Due to sigv4a -> sigv4 fallback, without a sigv4a implementation installed (it's optional) + * the credential should contain the ARN region, which is us-west-2, and not + * the us-east-1 region used by the client. + */ expect(result.request.headers["authorization"]).contains( `Credential=${credentials.accessKeyId}/${date}/${region}/s3-outposts/aws4_request` ); diff --git a/codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AddS3Config.java b/codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AddS3Config.java index 45a9c95a05b5..bf899d38dc3b 100644 --- a/codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AddS3Config.java +++ b/codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AddS3Config.java @@ -50,6 +50,7 @@ import software.amazon.smithy.typescript.codegen.TypeScriptDependency; import software.amazon.smithy.typescript.codegen.TypeScriptSettings; import software.amazon.smithy.typescript.codegen.TypeScriptWriter; +import software.amazon.smithy.typescript.codegen.auth.http.integration.AddHttpSigningPlugin; import software.amazon.smithy.typescript.codegen.integration.RuntimeClientPlugin; import software.amazon.smithy.typescript.codegen.integration.TypeScriptIntegration; import software.amazon.smithy.utils.ListUtils; @@ -81,6 +82,7 @@ public final class AddS3Config implements TypeScriptIntegration { @Override public List runAfter() { return List.of( + new AddHttpSigningPlugin().name(), AddBuiltinPlugins.class.getCanonicalName(), AddEndpointsPlugin.class.getCanonicalName() ); @@ -398,6 +400,11 @@ && containsInputMembers(m, o, BUCKET_ENDPOINT_INPUT_KEYS)) .withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "S3Express", HAS_MIDDLEWARE) .servicePredicate((m, s) -> isS3(s) && isEndpointsV2Service(s)) + .build(), + RuntimeClientPlugin.builder() + .withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "S3ExpressHttpSigning", + HAS_MIDDLEWARE) + .servicePredicate((m, s) -> isS3(s) && isEndpointsV2Service(s)) .build() ); } diff --git a/packages/core/src/submodules/httpAuthSchemes/aws_sdk/AwsSdkSigV4Signer.ts b/packages/core/src/submodules/httpAuthSchemes/aws_sdk/AwsSdkSigV4Signer.ts index de78134d8824..b0f7d5245d93 100644 --- a/packages/core/src/submodules/httpAuthSchemes/aws_sdk/AwsSdkSigV4Signer.ts +++ b/packages/core/src/submodules/httpAuthSchemes/aws_sdk/AwsSdkSigV4Signer.ts @@ -97,7 +97,24 @@ export class AwsSdkSigV4Signer implements HttpSigner { if (!HttpRequest.isInstance(httpRequest)) { throw new Error("The request is not an instance of `HttpRequest` and cannot be signed"); } - const { config, signer, signingRegion, signingName } = await validateSigningProperties(signingProperties); + + const validatedProps = await validateSigningProperties(signingProperties); + + const { config, signer } = validatedProps; + let { signingRegion, signingName } = validatedProps; + + const handlerExecutionContext = signingProperties.context as HandlerExecutionContext; + + if (handlerExecutionContext?.authSchemes?.length ?? 0 > 1) { + const [first, second] = handlerExecutionContext.authSchemes!; + // since this is not the sigv4a signer, we accept the secondary authscheme's signing data + // if the first authscheme is sigv4a and second is sigv4. + if (first?.name === "sigv4a" && second?.name === "sigv4") { + signingRegion = second?.signingRegion ?? signingRegion; + signingName = second?.signingName ?? signingName; + } + } + const signedRequest = await signer.sign(httpRequest, { signingDate: getSkewCorrectedDate(config.systemClockOffset), signingRegion: signingRegion, diff --git a/packages/core/src/submodules/httpAuthSchemes/aws_sdk/index.ts b/packages/core/src/submodules/httpAuthSchemes/aws_sdk/index.ts index 98b5ca4fecad..62172b787785 100644 --- a/packages/core/src/submodules/httpAuthSchemes/aws_sdk/index.ts +++ b/packages/core/src/submodules/httpAuthSchemes/aws_sdk/index.ts @@ -1,3 +1,3 @@ -export { AwsSdkSigV4Signer, AWSSDKSigV4Signer } from "./AwsSdkSigV4Signer"; +export { AwsSdkSigV4Signer, AWSSDKSigV4Signer, validateSigningProperties } from "./AwsSdkSigV4Signer"; export { AwsSdkSigV4ASigner } from "./AwsSdkSigV4ASigner"; export * from "./resolveAwsSdkSigV4Config"; diff --git a/packages/middleware-sdk-s3/package.json b/packages/middleware-sdk-s3/package.json index b63b84bd6d0b..55ee0dccffc2 100644 --- a/packages/middleware-sdk-s3/package.json +++ b/packages/middleware-sdk-s3/package.json @@ -23,14 +23,17 @@ }, "license": "Apache-2.0", "dependencies": { + "@aws-sdk/core": "*", "@aws-sdk/types": "*", "@aws-sdk/util-arn-parser": "*", + "@smithy/core": "^2.3.2", "@smithy/node-config-provider": "^3.1.4", "@smithy/protocol-http": "^4.1.0", "@smithy/signature-v4": "^4.1.0", "@smithy/smithy-client": "^3.1.12", "@smithy/types": "^3.3.0", "@smithy/util-config-provider": "^3.0.0", + "@smithy/util-middleware": "^3.0.3", "@smithy/util-stream": "^3.1.3", "@smithy/util-utf8": "^3.0.0", "tslib": "^2.6.2" diff --git a/packages/middleware-sdk-s3/src/s3-express/functions/s3ExpressHttpSigningMiddleware.ts b/packages/middleware-sdk-s3/src/s3-express/functions/s3ExpressHttpSigningMiddleware.ts new file mode 100644 index 000000000000..8d5509108440 --- /dev/null +++ b/packages/middleware-sdk-s3/src/s3-express/functions/s3ExpressHttpSigningMiddleware.ts @@ -0,0 +1,124 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ +import { httpSigningMiddlewareOptions } from "@smithy/core"; +import { HttpRequest, IHttpRequest } from "@smithy/protocol-http"; +import { + AuthScheme, + AwsCredentialIdentity, + ErrorHandler, + FinalizeHandler, + FinalizeHandlerArguments, + FinalizeHandlerOutput, + FinalizeRequestMiddleware, + HandlerExecutionContext, + Pluggable, + RequestSigner, + SelectedHttpAuthScheme, + SMITHY_CONTEXT_KEY, + SuccessHandler, +} from "@smithy/types"; +import { getSmithyContext } from "@smithy/util-middleware"; + +import { signS3Express } from "./signS3Express"; + +/** + * @internal + */ +interface HttpSigningMiddlewareSmithyContext extends Record { + selectedHttpAuthScheme?: SelectedHttpAuthScheme; +} + +/** + * @internal + */ +interface HttpSigningMiddlewareHandlerExecutionContext extends HandlerExecutionContext { + [SMITHY_CONTEXT_KEY]?: HttpSigningMiddlewareSmithyContext; +} + +const defaultErrorHandler: ErrorHandler = (signingProperties) => (error) => { + throw error; +}; + +const defaultSuccessHandler: SuccessHandler = ( + httpResponse: unknown, + signingProperties: Record +): void => {}; + +interface SigningProperties { + signingRegion: string; + signingDate: Date; + signingService: string; +} + +interface PreviouslyResolved { + signer: (authScheme?: AuthScheme | undefined) => Promise< + RequestSigner & { + signWithCredentials( + req: IHttpRequest, + identity: AwsCredentialIdentity, + opts?: Partial + ): Promise; + } + >; +} + +/** + * @internal + */ +export const s3ExpressHttpSigningMiddlewareOptions = httpSigningMiddlewareOptions; + +/** + * @internal + */ +export const s3ExpressHttpSigningMiddleware = + (config: PreviouslyResolved): FinalizeRequestMiddleware => + (next: FinalizeHandler, context: HttpSigningMiddlewareHandlerExecutionContext): FinalizeHandler => + async (args: FinalizeHandlerArguments): Promise> => { + if (!HttpRequest.isInstance(args.request)) { + return next(args); + } + + const smithyContext: HttpSigningMiddlewareSmithyContext = getSmithyContext(context); + const scheme = smithyContext.selectedHttpAuthScheme; + if (!scheme) { + throw new Error(`No HttpAuthScheme was selected: unable to sign request`); + } + const { + httpAuthOption: { signingProperties = {} }, + identity, + signer, + } = scheme; + + let request: IHttpRequest; + + if (context.s3ExpressIdentity) { + request = await signS3Express( + context.s3ExpressIdentity, + signingProperties as unknown as SigningProperties, + args.request, + await config.signer() + ); + } else { + request = await signer.sign(args.request, identity, signingProperties); + } + + const output = await next({ + ...args, + request, + }).catch((signer.errorHandler || defaultErrorHandler)(signingProperties)); + (signer.successHandler || defaultSuccessHandler)(output.response, signingProperties); + return output; + }; + +/** + * @internal + */ +export const getS3ExpressHttpSigningPlugin = (config: { + signer: (authScheme?: AuthScheme | undefined) => Promise; +}): Pluggable => ({ + applyToStack: (clientStack) => { + clientStack.addRelativeTo( + s3ExpressHttpSigningMiddleware(config as PreviouslyResolved), + httpSigningMiddlewareOptions + ); + }, +}); diff --git a/packages/middleware-sdk-s3/src/s3-express/functions/signS3Express.ts b/packages/middleware-sdk-s3/src/s3-express/functions/signS3Express.ts new file mode 100644 index 000000000000..9be783b8c161 --- /dev/null +++ b/packages/middleware-sdk-s3/src/s3-express/functions/signS3Express.ts @@ -0,0 +1,29 @@ +import type { AwsCredentialIdentity, HttpRequest as IHttpRequest } from "@smithy/types"; + +import { S3ExpressIdentity } from "../interfaces/S3ExpressIdentity"; + +export const signS3Express = async ( + s3ExpressIdentity: S3ExpressIdentity, + signingOptions: { + signingDate: Date; + signingRegion: string; + signingService: string; + }, + request: IHttpRequest, + sigV4MultiRegionSigner: { + signWithCredentials( + req: IHttpRequest, + identity: AwsCredentialIdentity, + opts?: Partial + ): Promise; + } +) => { + // the signer is expected to be SignatureV4MultiRegion for S3. + const signedRequest = await sigV4MultiRegionSigner.signWithCredentials(request, s3ExpressIdentity, {}); + + if (signedRequest.headers["X-Amz-Security-Token"] || signedRequest.headers["x-amz-security-token"]) { + throw new Error("X-Amz-Security-Token must not be set for s3-express requests."); + } + + return signedRequest; +}; diff --git a/packages/middleware-sdk-s3/src/s3-express/index.ts b/packages/middleware-sdk-s3/src/s3-express/index.ts index ef05b7d1748c..6d3b3fceb4d6 100644 --- a/packages/middleware-sdk-s3/src/s3-express/index.ts +++ b/packages/middleware-sdk-s3/src/s3-express/index.ts @@ -4,5 +4,10 @@ export { S3ExpressIdentityProviderImpl } from "./classes/S3ExpressIdentityProvid export { SignatureV4S3Express } from "./classes/SignatureV4S3Express"; export { NODE_DISABLE_S3_EXPRESS_SESSION_AUTH_OPTIONS } from "./constants"; export { getS3ExpressPlugin, s3ExpressMiddleware, s3ExpressMiddlewareOptions } from "./functions/s3ExpressMiddleware"; +export { + getS3ExpressHttpSigningPlugin, + s3ExpressHttpSigningMiddleware, + s3ExpressHttpSigningMiddlewareOptions, +} from "./functions/s3ExpressHttpSigningMiddleware"; export { S3ExpressIdentity } from "./interfaces/S3ExpressIdentity"; export { S3ExpressIdentityProvider } from "./interfaces/S3ExpressIdentityProvider";