Merge pull request #741 from aws/jasdel/cust/IMDSClient

Add EC2 IMDS API client and credentials provider

Makefile

@@ -49,7 +49,7 @@ gen-external-asserts:
 build:
 	go build -o /dev/null -tags ${ALL_TAGS} ${SDK_ALL_PKGS}
 
-unit: verify build test-protocols test-services
+unit: verify build test-protocols test-services test-ec2imds test-credentials test-config
 	@echo "go test SDK and vendor packages"
 	@go test -tags ${UNIT_TEST_TAGS} ${SDK_ALL_PKGS}
 
@@ -75,7 +75,16 @@ test-protocols:
 	./test_submodules.sh `pwd`/internal/protocoltest "go test -count 1 -run NONE ./..."
 
 test-services:
-	./test_submodules.sh `pwd`/service "go test -count 1 -run NONE ./..."
+	./test_submodules.sh `pwd`/service "go test -count 1 -tags "integration,ec2env" -run NONE ./..."
+
+test-ec2imds:
+	cd ec2imds && go test -run NONE -tags "integration,ec2env" ./... && go test -count 1 ./...
+
+test-credentials:
+	cd credentials && go test -run NONE -tags "integration,ec2env" ./... && go test -count 1 ./...
+
+test-config:
+	cd config && go test -run NONE -tags "integration,ec2env" ./... && go test -count 1 ./...
 
 mod_replace_local:
 	./mod_replace_local_submodules.sh `pwd` `pwd` `pwd`/../smithy-go

aws/config.go

@@ -1,5 +1,7 @@
 package aws
 
+import "github.com/awslabs/smithy-go/middleware"
+
 // A Config provides service configuration for service clients.
 type Config struct {
 	// The region to send requests to. This parameter is required and must
@@ -41,6 +43,11 @@ type Config struct {
 	// ConfigSources are the sources that were used to construct the Config.
 	// Allows for additional configuration to be loaded by clients.
 	ConfigSources []interface{}
+
+	// APIOptions provides the set of middleware mutations modify how the API
+	// client requests will be handled. This is useful for adding additional
+	// tracing data to a request, or changing behavior of the SDK's client.
+	APIOptions []func(*middleware.Stack) error
 }
 
 // NewConfig returns a new Config pointer that can be chained with builder

aws/request.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 )
 
+// TODO remove replace with smithy.CanceledError
+
 // RequestCanceledError is the error that will be returned by an API request
 // that was canceled. Requests given a Context may return this error when
 // canceled.

codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsGoDependency.java

@@ -51,6 +51,6 @@ protected static GoDependency aws(String relativePath, String alias) {
     }
 
     private static final class Versions {
-        private static final String AWS_SDK = "v0.24.1-0.20200921180648-50b89d38c63c";
+        private static final String AWS_SDK = "v0.0.0-20200923000934-8cf2e0ac6dea";
     }
 }

config/doc.go

@@ -0,0 +1,14 @@
+/*
+Package config provides utilities for loading configuration from multiple
+sources that can be used to configure the SDK's API clients, and utilities.
+
+The config package will load configuration from environment variables, AWS
+shared configuration file (~/.aws/config), and AWS shared credentials file
+(~/.aws/credentials).
+
+Use the LoadDefaultConfig to load configuration from all the SDK's supported
+sources, and resolve credentials using the SDK's default credential chain.
+
+* TODO Additional documentation needed.
+*/
+package config

config/go.mod

@@ -3,11 +3,14 @@ module github.com/aws/aws-sdk-go-v2/config
 go 1.15
 
 require (
-	github.com/aws/aws-sdk-go-v2 v0.24.1-0.20200921191144-68cec59c7dee
-	github.com/aws/aws-sdk-go-v2/credentials v0.0.0-20200921191144-68cec59c7dee
-	github.com/awslabs/smithy-go v0.0.0-20200914213924-b41e7bef5d4f
+	github.com/aws/aws-sdk-go-v2 v0.24.1-0.20200921180648-50b89d38c63c
+	github.com/aws/aws-sdk-go-v2/credentials v0.0.0-20200923000844-40e1b8c605ca
+	github.com/aws/aws-sdk-go-v2/ec2imds v0.0.0-20200923000844-40e1b8c605ca
+	github.com/awslabs/smithy-go v0.0.0-20200922192056-dab44aa99759
 )
 
-replace github.com/aws/aws-sdk-go-v2 => ../
-
-replace github.com/aws/aws-sdk-go-v2/credentials => ../credentials
+replace (
+	github.com/aws/aws-sdk-go-v2 => ../
+	github.com/aws/aws-sdk-go-v2/credentials => ../credentials
+	github.com/aws/aws-sdk-go-v2/ec2imds => ../ec2imds
+)

config/go.sum

@@ -1,11 +1,15 @@
-github.com/awslabs/smithy-go v0.0.0-20200914213924-b41e7bef5d4f h1:+ORlmI9IRG6Z/MDqHZfIgz/JvRWDVjUKS8FL/XXA1vc=
-github.com/awslabs/smithy-go v0.0.0-20200914213924-b41e7bef5d4f/go.mod h1:hPOQwnmBLHsUphH13tVSjQhTAFma0/0XoZGbBcOuABI=
+github.com/aws/aws-sdk-go-v2/service/sts v0.0.0-20200922201841-db749849ec30/go.mod h1:eBfZQDyojdkQZeXOPY3mgfCpUgbZCn8yg8WCx58vyLw=
+github.com/awslabs/smithy-go v0.0.0-20200917082847-627658712072/go.mod h1:hPOQwnmBLHsUphH13tVSjQhTAFma0/0XoZGbBcOuABI=
+github.com/awslabs/smithy-go v0.0.0-20200922192056-dab44aa99759 h1:jgGfVJaeAs+VN7pdR0YGvJhSUwHetVfGcisKwx6q4pU=
+github.com/awslabs/smithy-go v0.0.0-20200922192056-dab44aa99759/go.mod h1:hPOQwnmBLHsUphH13tVSjQhTAFma0/0XoZGbBcOuABI=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0=
 github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

config/provider.go

@@ -1,10 +1,13 @@
 package config
 
 import (
+	"context"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds"
 	"github.com/aws/aws-sdk-go-v2/credentials/processcreds"
+	"github.com/aws/aws-sdk-go-v2/ec2imds"
 )
 
 // SharedConfigProfileProvider provides access to the shared config profile
@@ -142,6 +145,38 @@ func GetRegion(configs Configs) (string, bool, error) {
 	return "", false, nil
 }
 
+// WithEC2IMDSRegion provides a RegionProvider that retrieves the region
+// from the EC2 Metadata service.
+//
+// TODO should this provider be added to the default config loading?
+type WithEC2IMDSRegion struct {
+	// If unset will be defaulted to Background context
+	Context context.Context
+
+	// If unset will default to generic EC2 IMDS client.
+	Client *ec2imds.Client
+}
+
+// GetRegion attempts to retrieve the region from EC2 Metadata service.
+func (p WithEC2IMDSRegion) GetRegion() (string, error) {
+	ctx := p.Context
+	if ctx == nil {
+		ctx = context.Background()
+	}
+
+	client := p.Client
+	if client == nil {
+		client = ec2imds.New(ec2imds.Options{})
+	}
+
+	result, err := p.Client.GetRegion(ctx, nil)
+	if err != nil {
+		return "", err
+	}
+
+	return result.Region, nil
+}
+
 // CredentialsProviderProvider provides access to the credentials external
 // configuration value.
 type CredentialsProviderProvider interface {
@@ -231,7 +266,8 @@ type ProcessCredentialOptions interface {
 	GetProcessCredentialOptions() (func(*processcreds.Options), bool, error)
 }
 
-// WithProcessCredentialOptions wraps a function and satisfies the EC2RoleCredentialOptions interface
+// WithProcessCredentialOptions wraps a function and satisfies the
+// ProcessCredentialOptions interface
 type WithProcessCredentialOptions func(*processcreds.Options)
 
 // GetProcessCredentialOptions returns the wrapped function
@@ -255,6 +291,37 @@ func GetProcessCredentialOptions(configs Configs) (f func(*processcreds.Options)
 	return f, found, err
 }
 
+// EC2RoleCredentialProviderOptions is an interface for retrieving a function
+// for setting the ec2rolecreds.Provider options.
+type EC2RoleCredentialProviderOptions interface {
+	GetEC2RoleCredentialProviderOptions() (func(*ec2rolecreds.Options), bool, error)
+}
+
+// WithEC2RoleCredentialProviderOptions wraps a function and satisfies the
+// EC2RoleCredentialProviderOptions interface
+type WithEC2RoleCredentialProviderOptions func(*ec2rolecreds.Options)
+
+// GetEC2RoleCredentialProviderOptions returns the wrapped function
+func (w WithEC2RoleCredentialProviderOptions) GetEC2RoleCredentialProviderOptions() (func(*ec2rolecreds.Options), bool, error) {
+	return w, true, nil
+}
+
+// GetEC2RoleCredentialProviderOptions searches the slice of configs and returns the first function found
+func GetEC2RoleCredentialProviderOptions(configs Configs) (f func(*ec2rolecreds.Options), found bool, err error) {
+	for _, config := range configs {
+		if p, ok := config.(EC2RoleCredentialProviderOptions); ok {
+			f, found, err = p.GetEC2RoleCredentialProviderOptions()
+			if err != nil {
+				return nil, false, err
+			}
+			if found {
+				break
+			}
+		}
+	}
+	return f, found, err
+}
+
 // DefaultRegionProvider is an interface for retrieving a default region if a region was not resolved from other sources
 type DefaultRegionProvider interface {
 	GetDefaultRegion() (string, bool, error)

config/resolve_credentials.go

@@ -3,10 +3,13 @@ package config
 import (
 	"fmt"
 	"net/url"
+	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds"
 	"github.com/aws/aws-sdk-go-v2/credentials/processcreds"
+	"github.com/aws/aws-sdk-go-v2/ec2imds"
 )
 
 const (
@@ -213,6 +216,36 @@ func resolveCredsFromSource(cfg *aws.Config, envConfig *EnvConfig, sharedCfg *Sh
 	return nil
 }
 
+func resolveEC2RoleCredentials(cfg *aws.Config, configs Configs) error {
+	optFns := make([]func(*ec2rolecreds.Options), 0, 2)
+
+	optFn, found, err := GetEC2RoleCredentialProviderOptions(configs)
+	if err != nil {
+		return err
+	}
+	if found {
+		optFns = append(optFns, optFn)
+	}
+
+	optFns = append(optFns, func(o *ec2rolecreds.Options) {
+		// Only define a client from config if not already defined.
+		if o.Client != nil {
+			o.Client = ec2imds.New(ec2imds.Options{
+				HTTPClient: cfg.HTTPClient,
+				Retryer:    cfg.Retryer,
+			})
+		}
+	})
+
+	provider := ec2rolecreds.New(ec2rolecreds.Options{
+		ExpiryWindow: 5 * time.Minute,
+	}, optFns...)
+
+	cfg.Credentials = provider
+
+	return nil
+}
+
 func getAWSConfigSources(configs Configs) (*EnvConfig, *SharedConfig, Configs) {
 	var (
 		envConfig    *EnvConfig

config/resolve_credentials_stubs.go

@@ -10,10 +10,6 @@ func resolveHTTPCredProvider(cfg *aws.Config, url, authToken string, configs Con
 	return fmt.Errorf("endpoint credential provider is not currently supported")
 }
 
-func resolveEC2RoleCredentials(cfg *aws.Config, configs Configs) error {
-	return fmt.Errorf("ec2 role credential provider is not currently supported")
-}
-
 func assumeWebIdentity(cfg *aws.Config, filepath string, roleARN, sessionName string, configs Configs) error {
 	return fmt.Errorf("assume web identity role is not currently supported")
 }

credentials/ec2rolecreds/doc.go

@@ -0,0 +1,60 @@
+/*
+Package ec2rolecreds provides the credentials provider implementation for
+retrieving AWS credentials from Amazon EC2 Instance Roles via Amazon EC2 IMDS.
+
+Concurrency and caching
+
+The Provider is not safe to be used concurrently, and does not provide any
+caching of credentials retrieved. You should wrap the Provider with a
+`aws.CredentialsCache` to provide concurrency safety, and caching of
+credentials.
+
+Loading credentials with the SDK's AWS Config
+
+The EC2 Instance role credentials provider will automatically be the resolved
+credential provider int he credential chain if no other credential provider is
+resolved first.
+
+To explicitly instruct the SDK's credentials resolving to use the EC2 Instance
+role for credentials, you specify a `credentials_source` property in the config
+profile the SDK will load.
+
+    [default]
+    credential_source = Ec2InstanceMetadata
+
+Loading credentials with the Provider directly
+
+Another way to use the EC2 Instance role credentials provider is to create it
+directly and assign it as the credentials provider for an API client.
+
+The following example creates a credentials provider for a command, and wraps
+it with the CredentialsCache before assigning the provider to the Amazon S3 API
+client's Credentials option.
+
+	provider := ec2imds.New(ec2imds.Options{})
+
+    // Create the service client value configured for credentials.
+    svc := s3.New(s3.Options{
+		Credentials: &aws.CredentialsCache{Provider: provider},
+	})
+
+If you need more control, you can set the configuration options on the
+credentials provider using the ec2imds.Options type to configure the EC2 IMDS
+API Client and ExpiryWindow of the retrieved credentials.
+
+	provider := ec2imds.New(ec2imds.Options{
+		// See ec2imds.Options type's documentation for more options available.
+		Client: ec2imds.New(Options{
+			HTTPClient: customHTTPClient,
+		}),
+
+		// Modify how soon credentials expire prior to their original expiry time.
+		ExpiryWindow: 5 * time.Minute,
+	})
+
+EC2 IMDS API Client
+
+See the github.com/aws/aws-sdk-go-v2/ec2imds module for more details on
+configuring the client, and options available.
+*/
+package ec2rolecreds

credentials/ec2rolecreds/integration_test.go

@@ -0,0 +1,23 @@
+// +build integration,ec2env
+
+package ec2rolecreds
+
+import (
+	"context"
+	"testing"
+)
+
+func TestInteg_RetrieveCredentials(t *testing.T) {
+	provider := New(Options{})
+
+	creds, err := provider.Retrieve(context.Background())
+	if err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+
+	if !creds.HasKeys() {
+		t.Errorf("expect credential values, got none")
+	}
+
+	t.Logf("AccessKey: %v", creds.AccessKeyID)
+}