From 2ccaeaa772151fe39ea90f47b33060114081ffe7 Mon Sep 17 00:00:00 2001 From: David Turner Date: Sat, 5 May 2018 17:48:43 +0100 Subject: [PATCH 01/21] Add support for AWS session tokens AWS supports the creation and use of credentials that are only valid for a fixed period of time. These credentials comprise three parts: the usual access key and secret key, together with a session token. This commit adds support for these three-part credentials to the EC2 discovery plugin and the S3 repository plugin. Note that session tokens are only valid for a limited period of time and yet there is no mechanism for refreshing or rotating them when they expire without restarting Elasticsearch. Nonetheless, this feature is already useful for nodes that need only run for a few days, such as for training, testing or evaluation. #29135 tracks the work towards allowing these credentials to be refreshed at runtime. Resolves #16428 --- docs/CHANGELOG.asciidoc | 3 + docs/plugins/discovery-ec2.asciidoc | 5 + docs/plugins/repository-s3.asciidoc | 5 + .../discovery/ec2/AwsEc2Service.java | 3 + .../discovery/ec2/AwsEc2ServiceImpl.java | 42 ++++-- .../discovery/ec2/Ec2DiscoveryPlugin.java | 1 + .../discovery/ec2/AwsEc2ServiceImplTests.java | 59 ++++++-- .../Ec2DiscoveryClusterFormationTests.java | 3 + .../repositories/s3/InternalAwsS3Service.java | 4 +- .../repositories/s3/S3ClientSettings.java | 29 +++- .../repositories/s3/S3RepositoryPlugin.java | 1 + .../s3/S3ClientSettingsTests.java | 129 ++++++++++++++++++ .../common/settings/SettingsException.java | 4 + 13 files changed, 260 insertions(+), 28 deletions(-) create mode 100644 plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java diff --git a/docs/CHANGELOG.asciidoc b/docs/CHANGELOG.asciidoc index 380c7288e8424..066ac9301b437 100644 --- a/docs/CHANGELOG.asciidoc +++ b/docs/CHANGELOG.asciidoc @@ -149,6 +149,9 @@ option. ({pull}30140[#29658]) A new analysis plugin called `analysis_nori` that exposes the Lucene Korean analysis module. ({pull}30397[#30397]) +The `discovery-ec2` and `repository-s3` plugins now have support for AWS +session credentials. ({pull}TODO[#TODO]) + [float] === Enhancements diff --git a/docs/plugins/discovery-ec2.asciidoc b/docs/plugins/discovery-ec2.asciidoc index 2e2bc9cf268fa..1d77439cfea30 100644 --- a/docs/plugins/discovery-ec2.asciidoc +++ b/docs/plugins/discovery-ec2.asciidoc @@ -46,6 +46,11 @@ Those that must be stored in the keystore are marked as `Secure`. An s3 secret key. The `access_key` setting must also be specified. (Secure) +`session_token`:: + + An s3 session token. The `access_key` and `secret_key` settings must also + be specified. (Secure) + `endpoint`:: The ec2 service endpoint to connect to. This will be automatically diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc index bff64ebdc9186..5071614c113a6 100644 --- a/docs/plugins/repository-s3.asciidoc +++ b/docs/plugins/repository-s3.asciidoc @@ -73,6 +73,11 @@ are marked as `Secure`. An s3 secret key. The `access_key` setting must also be specified. (Secure) +`session_token`:: + + An s3 session token. The `access_key` and `secret_key` settings must also + be specified. (Secure) + `endpoint`:: The s3 service endpoint to connect to. This will be automatically diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2Service.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2Service.java index 880be6c037323..1d3b040e91b01 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2Service.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2Service.java @@ -52,6 +52,9 @@ class HostType { /** The secret key (ie password) for connecting to ec2. */ Setting SECRET_KEY_SETTING = SecureSetting.secureString("discovery.ec2.secret_key", null); + /** The (optional) session token for connecting to ec2. */ + Setting SESSION_TOKEN_SETTING = SecureSetting.secureString("discovery.ec2.session_token", null); + /** An override for the ec2 endpoint to connect to. */ Setting ENDPOINT_SETTING = new Setting<>("discovery.ec2.endpoint", "", s -> s.toLowerCase(Locale.ROOT), Property.NodeScope); diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java index 3b5b955260e6d..22b08d3542a09 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java @@ -26,8 +26,11 @@ import com.amazonaws.AmazonClientException; import com.amazonaws.AmazonWebServiceRequest; import com.amazonaws.ClientConfiguration; +import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.auth.BasicSessionCredentials; import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; import com.amazonaws.http.IdleConnectionReaper; import com.amazonaws.internal.StaticCredentialsProvider; @@ -35,13 +38,18 @@ import com.amazonaws.services.ec2.AmazonEC2; import com.amazonaws.services.ec2.AmazonEC2Client; import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.Randomness; import org.elasticsearch.common.Strings; import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsException; class AwsEc2ServiceImpl extends AbstractComponent implements AwsEc2Service, Closeable { + private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(AwsEc2ServiceImpl.class)); public static final String EC2_METADATA_URL = "http://169.254.169.254/latest/meta-data/"; @@ -67,20 +75,38 @@ public synchronized AmazonEC2 client() { } protected static AWSCredentialsProvider buildCredentials(Logger logger, Settings settings) { - AWSCredentialsProvider credentials; - try (SecureString key = ACCESS_KEY_SETTING.get(settings); - SecureString secret = SECRET_KEY_SETTING.get(settings)) { + SecureString secret = SECRET_KEY_SETTING.get(settings); + SecureString sessionToken = SESSION_TOKEN_SETTING.get(settings)) { if (key.length() == 0 && secret.length() == 0) { + if (sessionToken.length() > 0) { + throw new SettingsException("Setting [{}] is set but [{}] and [{}] are not", + SESSION_TOKEN_SETTING.getKey(), ACCESS_KEY_SETTING.getKey(), SECRET_KEY_SETTING.getKey()); + } + logger.debug("Using either environment variables, system properties or instance profile credentials"); - credentials = new DefaultAWSCredentialsProviderChain(); + return new DefaultAWSCredentialsProviderChain(); } else { - logger.debug("Using basic key/secret credentials"); - credentials = new StaticCredentialsProvider(new BasicAWSCredentials(key.toString(), secret.toString())); + if (key.length() == 0) { + DEPRECATION_LOGGER.deprecated("Setting [{}] is set but [{}] is not, which will be unsupported in future", + SECRET_KEY_SETTING.getKey(), ACCESS_KEY_SETTING.getKey()); + } + if (secret.length() == 0) { + DEPRECATION_LOGGER.deprecated("Setting [{}] is set but [{}] is not, which will be unsupported in future", + ACCESS_KEY_SETTING.getKey(), SECRET_KEY_SETTING.getKey()); + } + + final AWSCredentials credentials; + if (sessionToken.length() == 0) { + logger.debug("Using basic key/secret credentials"); + credentials = new BasicAWSCredentials(key.toString(), secret.toString()); + } else { + logger.debug("Using basic session credentials"); + credentials = new BasicSessionCredentials(key.toString(), secret.toString(), sessionToken.toString()); + } + return new AWSStaticCredentialsProvider(credentials); } } - - return credentials; } protected static ClientConfiguration buildConfiguration(Logger logger, Settings settings) { diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java index 28d563e6a9ca6..b1078cdd88308 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java @@ -106,6 +106,7 @@ public List> getSettings() { // Register EC2 discovery settings: discovery.ec2 AwsEc2Service.ACCESS_KEY_SETTING, AwsEc2Service.SECRET_KEY_SETTING, + AwsEc2Service.SESSION_TOKEN_SETTING, AwsEc2Service.ENDPOINT_SETTING, AwsEc2Service.PROTOCOL_SETTING, AwsEc2Service.PROXY_HOST_SETTING, diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImplTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImplTests.java index 06693bbff11ac..7756191100166 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImplTests.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImplTests.java @@ -23,15 +23,13 @@ import com.amazonaws.Protocol; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.BasicSessionCredentials; import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; import org.elasticsearch.common.settings.MockSecureSettings; -import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.discovery.ec2.AwsEc2Service; -import org.elasticsearch.discovery.ec2.AwsEc2ServiceImpl; +import org.elasticsearch.common.settings.SettingsException; import org.elasticsearch.test.ESTestCase; -import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; @@ -44,17 +42,56 @@ public void testAWSCredentialsWithSystemProviders() { } public void testAWSCredentialsWithElasticsearchAwsSettings() { - MockSecureSettings secureSettings = new MockSecureSettings(); + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("discovery.ec2.access_key", "aws_key"); + secureSettings.setString("discovery.ec2.secret_key", "aws_secret"); + final AWSCredentials credentials = AwsEc2ServiceImpl.buildCredentials(logger, + Settings.builder().setSecureSettings(secureSettings).build()).getCredentials(); + assertThat(credentials.getAWSAccessKeyId(), is("aws_key")); + assertThat(credentials.getAWSSecretKey(), is("aws_secret")); + } + + public void testAWSSessionCredentialsWithElasticsearchAwsSettings() { + final MockSecureSettings secureSettings = new MockSecureSettings(); secureSettings.setString("discovery.ec2.access_key", "aws_key"); secureSettings.setString("discovery.ec2.secret_key", "aws_secret"); - Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); - launchAWSCredentialsWithElasticsearchSettingsTest(settings, "aws_key", "aws_secret"); + secureSettings.setString("discovery.ec2.session_token", "aws_session_token"); + final BasicSessionCredentials credentials = (BasicSessionCredentials) AwsEc2ServiceImpl.buildCredentials(logger, + Settings.builder().setSecureSettings(secureSettings).build()).getCredentials(); + assertThat(credentials.getAWSAccessKeyId(), is("aws_key")); + assertThat(credentials.getAWSSecretKey(), is("aws_secret")); + assertThat(credentials.getSessionToken(), is("aws_session_token")); + } + + public void testDeprecationOfLoneAccessKey() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("discovery.ec2.access_key", "aws_key"); + final AWSCredentials credentials = AwsEc2ServiceImpl.buildCredentials(logger, + Settings.builder().setSecureSettings(secureSettings).build()).getCredentials(); + assertThat(credentials.getAWSAccessKeyId(), is("aws_key")); + assertThat(credentials.getAWSSecretKey(), is("")); + assertSettingDeprecationsAndWarnings(new String[]{}, + "Setting [discovery.ec2.access_key] is set but [discovery.ec2.secret_key] is not, which will be unsupported in future"); + } + + public void testDeprecationOfLoneSecretKey() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("discovery.ec2.secret_key", "aws_secret"); + final AWSCredentials credentials = AwsEc2ServiceImpl.buildCredentials(logger, + Settings.builder().setSecureSettings(secureSettings).build()).getCredentials(); + assertThat(credentials.getAWSAccessKeyId(), is("")); + assertThat(credentials.getAWSSecretKey(), is("aws_secret")); + assertSettingDeprecationsAndWarnings(new String[]{}, + "Setting [discovery.ec2.secret_key] is set but [discovery.ec2.access_key] is not, which will be unsupported in future"); } - protected void launchAWSCredentialsWithElasticsearchSettingsTest(Settings settings, String expectedKey, String expectedSecret) { - AWSCredentials credentials = AwsEc2ServiceImpl.buildCredentials(logger, settings).getCredentials(); - assertThat(credentials.getAWSAccessKeyId(), is(expectedKey)); - assertThat(credentials.getAWSSecretKey(), is(expectedSecret)); + public void testRejectionOfLoneSessionToken() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("discovery.ec2.session_token", "aws_session_token"); + SettingsException e = expectThrows(SettingsException.class, () -> AwsEc2ServiceImpl.buildCredentials(logger, + Settings.builder().setSecureSettings(secureSettings).build())); + assertThat(e.getMessage(), is( + "Setting [discovery.ec2.session_token] is set but [discovery.ec2.access_key] and [discovery.ec2.secret_key] are not")); } public void testAWSDefaultConfiguration() { diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryClusterFormationTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryClusterFormationTests.java index 49fd9de71ecfa..66e410a3ae62e 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryClusterFormationTests.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryClusterFormationTests.java @@ -82,6 +82,9 @@ protected Settings nodeSettings(int nodeOrdinal) { MockSecureSettings secureSettings = new MockSecureSettings(); secureSettings.setString(AwsEc2Service.ACCESS_KEY_SETTING.getKey(), "some_access"); secureSettings.setString(AwsEc2Service.SECRET_KEY_SETTING.getKey(), "some_secret"); + if (randomBoolean()) { + secureSettings.setString(AwsEc2Service.SESSION_TOKEN_SETTING.getKey(), "some_token"); + } return Settings.builder().put(super.nodeSettings(nodeOrdinal)) .put(DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING.getKey(), "ec2") .put("path.logs", resolve) diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java index d70ed9ea9aa8b..bc9bd3237f9b6 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java @@ -111,9 +111,7 @@ static ClientConfiguration buildConfiguration(S3ClientSettings clientSettings) { // pkg private for tests static AWSCredentialsProvider buildCredentials(Logger logger, DeprecationLogger deprecationLogger, S3ClientSettings clientSettings, Settings repositorySettings) { - - - BasicAWSCredentials credentials = clientSettings.credentials; + AWSCredentials credentials = clientSettings.credentials; if (S3Repository.ACCESS_KEY_SETTING.exists(repositorySettings)) { if (S3Repository.SECRET_KEY_SETTING.exists(repositorySettings) == false) { throw new IllegalArgumentException("Repository setting [" + S3Repository.ACCESS_KEY_SETTING.getKey() + diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java index 4d32d2518fff1..d324375e6ec87 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java @@ -27,7 +27,9 @@ import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; +import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.auth.BasicSessionCredentials; import org.elasticsearch.common.settings.SecureSetting; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Setting; @@ -51,6 +53,10 @@ class S3ClientSettings { static final Setting.AffixSetting SECRET_KEY_SETTING = Setting.affixKeySetting(PREFIX, "secret_key", key -> SecureSetting.secureString(key, null)); + /** The (optional) session token for connecting to s3. */ + static final Setting.AffixSetting SESSION_TOKEN_SETTING = Setting.affixKeySetting(PREFIX, "session_token", + key -> SecureSetting.secureString(key, null)); + /** An override for the s3 endpoint to connect to. */ static final Setting.AffixSetting ENDPOINT_SETTING = Setting.affixKeySetting(PREFIX, "endpoint", key -> new Setting<>(key, "", s -> s.toLowerCase(Locale.ROOT), Property.NodeScope)); @@ -88,7 +94,7 @@ class S3ClientSettings { key -> Setting.boolSetting(key, ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope)); /** Credentials to authenticate with s3. */ - final BasicAWSCredentials credentials; + final AWSCredentials credentials; /** The s3 endpoint the client should talk to, or empty string to use the default. */ final String endpoint; @@ -119,7 +125,7 @@ class S3ClientSettings { /** Whether the s3 client should use an exponential backoff retry policy. */ final boolean throttleRetries; - private S3ClientSettings(BasicAWSCredentials credentials, String endpoint, Protocol protocol, + private S3ClientSettings(AWSCredentials credentials, String endpoint, Protocol protocol, String proxyHost, int proxyPort, String proxyUsername, String proxyPassword, int readTimeoutMillis, int maxRetries, boolean throttleRetries) { this.credentials = credentials; @@ -158,17 +164,28 @@ static Map load(Settings settings) { static S3ClientSettings getClientSettings(Settings settings, String clientName) { try (SecureString accessKey = getConfigValue(settings, clientName, ACCESS_KEY_SETTING); SecureString secretKey = getConfigValue(settings, clientName, SECRET_KEY_SETTING); + SecureString sessionToken = getConfigValue(settings, clientName, SESSION_TOKEN_SETTING); SecureString proxyUsername = getConfigValue(settings, clientName, PROXY_USERNAME_SETTING); SecureString proxyPassword = getConfigValue(settings, clientName, PROXY_PASSWORD_SETTING)) { - BasicAWSCredentials credentials = null; + final AWSCredentials credentials; if (accessKey.length() != 0) { if (secretKey.length() != 0) { - credentials = new BasicAWSCredentials(accessKey.toString(), secretKey.toString()); + if (sessionToken.length() != 0) { + credentials = new BasicSessionCredentials(accessKey.toString(), secretKey.toString(), sessionToken.toString()); + } else { + credentials = new BasicAWSCredentials(accessKey.toString(), secretKey.toString()); + } } else { throw new IllegalArgumentException("Missing secret key for s3 client [" + clientName + "]"); } - } else if (secretKey.length() != 0) { - throw new IllegalArgumentException("Missing access key for s3 client [" + clientName + "]"); + } else { + if (secretKey.length() != 0) { + throw new IllegalArgumentException("Missing access key for s3 client [" + clientName + "]"); + } + if (sessionToken.length() != 0) { + throw new IllegalArgumentException("Missing access key and secret key for s3 client [" + clientName + "]"); + } + credentials = null; } return new S3ClientSettings( credentials, diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java index 010c4b92c21a0..9c3a04d0a3f23 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java @@ -82,6 +82,7 @@ public List> getSettings() { // named s3 client configuration settings S3ClientSettings.ACCESS_KEY_SETTING, S3ClientSettings.SECRET_KEY_SETTING, + S3ClientSettings.SESSION_TOKEN_SETTING, S3ClientSettings.ENDPOINT_SETTING, S3ClientSettings.PROTOCOL_SETTING, S3ClientSettings.PROXY_HOST_SETTING, diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java new file mode 100644 index 0000000000000..6bc0d901738ae --- /dev/null +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java @@ -0,0 +1,129 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.repositories.s3; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.auth.BasicSessionCredentials; +import org.elasticsearch.common.settings.MockSecureSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; + +import java.util.Map; + +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.isEmptyString; +import static org.hamcrest.Matchers.nullValue; + +public class S3ClientSettingsTests extends ESTestCase { + public void testThereIsADefaultClientByDefault() { + final Map settings = S3ClientSettings.load(Settings.EMPTY); + assertThat(settings.keySet(), contains("default")); + + final S3ClientSettings defaultSettings = settings.get("default"); + assertThat(defaultSettings.credentials, nullValue()); + assertThat(defaultSettings.endpoint, isEmptyString()); + assertThat(defaultSettings.protocol, is(Protocol.HTTPS)); + assertThat(defaultSettings.proxyHost, isEmptyString()); + assertThat(defaultSettings.proxyPort, is(80)); + assertThat(defaultSettings.proxyUsername, isEmptyString()); + assertThat(defaultSettings.proxyPassword, isEmptyString()); + assertThat(defaultSettings.readTimeoutMillis, is(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT)); + assertThat(defaultSettings.maxRetries, is(ClientConfiguration.DEFAULT_RETRY_POLICY.getMaxErrorRetry())); + assertThat(defaultSettings.throttleRetries, is(ClientConfiguration.DEFAULT_THROTTLE_RETRIES)); + } + + public void testDefaultClientSettingsCanBeSet() { + final Map settings = S3ClientSettings.load(Settings.builder() + .put("s3.client.default.max_retries", 10).build()); + assertThat(settings.keySet(), contains("default")); + + final S3ClientSettings defaultSettings = settings.get("default"); + assertThat(defaultSettings.maxRetries, is(10)); + } + + public void testNondefaultClientCreatedBySettingItsSettings() { + final Map settings = S3ClientSettings.load(Settings.builder() + .put("s3.client.another_client.max_retries", 10).build()); + assertThat(settings.keySet(), contains("default", "another_client")); + + final S3ClientSettings defaultSettings = settings.get("default"); + assertThat(defaultSettings.maxRetries, is(ClientConfiguration.DEFAULT_RETRY_POLICY.getMaxErrorRetry())); + + final S3ClientSettings anotherClientSettings = settings.get("another_client"); + assertThat(anotherClientSettings.maxRetries, is(10)); + } + + public void testRejectionOfLoneAccessKey() { + final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("s3.client.default.access_key", "aws_key"); + S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()); + }); + assertThat(e.getMessage(), is("Missing secret key for s3 client [default]")); + } + + public void testRejectionOfLoneSecretKey() { + final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("s3.client.default.secret_key", "aws_key"); + S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()); + }); + assertThat(e.getMessage(), is("Missing access key for s3 client [default]")); + } + + public void testRejectionOfLoneSessionToken() { + final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("s3.client.default.session_token", "aws_key"); + S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()); + }); + assertThat(e.getMessage(), is("Missing access key and secret key for s3 client [default]")); + } + + public void testCredentialsTypeWithAccessKeyAndSecretKey() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("s3.client.default.access_key", "access_key"); + secureSettings.setString("s3.client.default.secret_key", "secret_key"); + final Map settings = S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()); + final S3ClientSettings defaultSettings = settings.get("default"); + BasicAWSCredentials credentials = (BasicAWSCredentials) defaultSettings.credentials; + assertThat(credentials.getAWSAccessKeyId(), is("access_key")); + assertThat(credentials.getAWSSecretKey(), is("secret_key")); + } + + public void testCredentialsTypeWithAccessKeyAndSecretKeyAndSessionToken() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("s3.client.default.access_key", "access_key"); + secureSettings.setString("s3.client.default.secret_key", "secret_key"); + secureSettings.setString("s3.client.default.session_token", "session_token"); + final Map settings = S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()); + final S3ClientSettings defaultSettings = settings.get("default"); + BasicSessionCredentials credentials = (BasicSessionCredentials) defaultSettings.credentials; + assertThat(credentials.getAWSAccessKeyId(), is("access_key")); + assertThat(credentials.getAWSSecretKey(), is("secret_key")); + assertThat(credentials.getSessionToken(), is("session_token")); + } +} diff --git a/server/src/main/java/org/elasticsearch/common/settings/SettingsException.java b/server/src/main/java/org/elasticsearch/common/settings/SettingsException.java index f7d4843c1c03e..ad5f56d7fc0f3 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/SettingsException.java +++ b/server/src/main/java/org/elasticsearch/common/settings/SettingsException.java @@ -42,4 +42,8 @@ public SettingsException(String message, Throwable cause) { public SettingsException(StreamInput in) throws IOException { super(in); } + + public SettingsException(String msg, Object... args) { + super(msg, args); + } } From 0fbe0390094fb8cac0e5381592af72e6302c247c Mon Sep 17 00:00:00 2001 From: David Turner Date: Sun, 6 May 2018 13:02:13 +0100 Subject: [PATCH 02/21] Add PR number to changelog --- docs/CHANGELOG.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/CHANGELOG.asciidoc b/docs/CHANGELOG.asciidoc index 066ac9301b437..2d7be97c79935 100644 --- a/docs/CHANGELOG.asciidoc +++ b/docs/CHANGELOG.asciidoc @@ -150,7 +150,7 @@ A new analysis plugin called `analysis_nori` that exposes the Lucene Korean analysis module. ({pull}30397[#30397]) The `discovery-ec2` and `repository-s3` plugins now have support for AWS -session credentials. ({pull}TODO[#TODO]) +session credentials. ({pull}30414[#30414]) [float] === Enhancements From c93e56842e8083e6d6018cb9bd0924ecd7c86ec6 Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 22 May 2018 17:08:34 +0100 Subject: [PATCH 03/21] discovery-ec2 docs shouldn't refer to S3 --- docs/plugins/discovery-ec2.asciidoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/plugins/discovery-ec2.asciidoc b/docs/plugins/discovery-ec2.asciidoc index 1d77439cfea30..7e785de79b7b1 100644 --- a/docs/plugins/discovery-ec2.asciidoc +++ b/docs/plugins/discovery-ec2.asciidoc @@ -40,15 +40,15 @@ Those that must be stored in the keystore are marked as `Secure`. `access_key`:: - An s3 access key. The `secret_key` setting must also be specified. (Secure) + An ec2 access key. The `secret_key` setting must also be specified. (Secure) `secret_key`:: - An s3 secret key. The `access_key` setting must also be specified. (Secure) + An ec2 secret key. The `access_key` setting must also be specified. (Secure) `session_token`:: - An s3 session token. The `access_key` and `secret_key` settings must also + An ec2 session token. The `access_key` and `secret_key` settings must also be specified. (Secure) `endpoint`:: From ee41de633714c0797dd2823595d4a46afa6c68b6 Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 22 May 2018 17:10:21 +0100 Subject: [PATCH 04/21] Reduce scope of expectThrows --- .../s3/S3ClientSettingsTests.java | 24 +++++++------------ 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java index 6bc0d901738ae..e629f43f8a3d3 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java @@ -74,32 +74,26 @@ public void testNondefaultClientCreatedBySettingItsSettings() { } public void testRejectionOfLoneAccessKey() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("s3.client.default.access_key", "aws_key"); final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> { - final MockSecureSettings secureSettings = new MockSecureSettings(); - secureSettings.setString("s3.client.default.access_key", "aws_key"); - S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()); - }); + () -> S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build())); assertThat(e.getMessage(), is("Missing secret key for s3 client [default]")); } public void testRejectionOfLoneSecretKey() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("s3.client.default.secret_key", "aws_key"); final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> { - final MockSecureSettings secureSettings = new MockSecureSettings(); - secureSettings.setString("s3.client.default.secret_key", "aws_key"); - S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()); - }); + () -> S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build())); assertThat(e.getMessage(), is("Missing access key for s3 client [default]")); } public void testRejectionOfLoneSessionToken() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("s3.client.default.session_token", "aws_key"); final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> { - final MockSecureSettings secureSettings = new MockSecureSettings(); - secureSettings.setString("s3.client.default.session_token", "aws_key"); - S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()); - }); + () -> S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build())); assertThat(e.getMessage(), is("Missing access key and secret key for s3 client [default]")); } From 85737f6829ec7ed54d0562205908d0f473ed40a3 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 7 Jun 2018 09:44:18 +0100 Subject: [PATCH 05/21] Support injection of session token from the environment --- plugins/repository-s3/qa/amazon-s3/build.gradle | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/repository-s3/qa/amazon-s3/build.gradle b/plugins/repository-s3/qa/amazon-s3/build.gradle index 5e288899021a1..a0ecea4d67055 100644 --- a/plugins/repository-s3/qa/amazon-s3/build.gradle +++ b/plugins/repository-s3/qa/amazon-s3/build.gradle @@ -43,8 +43,9 @@ String s3AccessKey = System.getenv("amazon_s3_access_key") String s3SecretKey = System.getenv("amazon_s3_secret_key") String s3Bucket = System.getenv("amazon_s3_bucket") String s3BasePath = System.getenv("amazon_s3_base_path") +String s3SessionToken = System.getenv("amazon_s3_session_token") -if (!s3AccessKey && !s3SecretKey && !s3Bucket && !s3BasePath) { +if (!s3AccessKey && !s3SecretKey && !s3Bucket && !s3BasePath && !s3SessionToken) { s3AccessKey = 's3_integration_test_access_key' s3SecretKey = 's3_integration_test_secret_key' s3Bucket = 'bucket_test' @@ -72,6 +73,9 @@ processTestResources { integTestCluster { keystoreSetting 's3.client.integration_test.access_key', s3AccessKey keystoreSetting 's3.client.integration_test.secret_key', s3SecretKey + if (s3SessionToken) { + keystoreSetting 's3.client.integration_test.session_token', s3SessionToken + } if (useFixture) { dependsOn s3Fixture From 2eea8291dde71058e534c95f1c3f4490c683948e Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 7 Jun 2018 11:41:13 +0100 Subject: [PATCH 06/21] Extend QA tests to test using both temporary and permanent credentials --- .../repository-s3/qa/amazon-s3/build.gradle | 65 +++++-- .../repositories/s3/AmazonS3Fixture.java | 7 +- .../repositories/s3/AmazonS3TestServer.java | 34 +++- ...> 10_repository_permanent_credentials.yml} | 47 ++--- .../20_repository_temporary_credentials.yml | 184 ++++++++++++++++++ 5 files changed, 288 insertions(+), 49 deletions(-) rename plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/{10_repository.yml => 10_repository_permanent_credentials.yml} (71%) create mode 100644 plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_temporary_credentials.yml diff --git a/plugins/repository-s3/qa/amazon-s3/build.gradle b/plugins/repository-s3/qa/amazon-s3/build.gradle index a0ecea4d67055..865c3407faacd 100644 --- a/plugins/repository-s3/qa/amazon-s3/build.gradle +++ b/plugins/repository-s3/qa/amazon-s3/build.gradle @@ -39,17 +39,37 @@ forbiddenApisTest { boolean useFixture = false -String s3AccessKey = System.getenv("amazon_s3_access_key") -String s3SecretKey = System.getenv("amazon_s3_secret_key") -String s3Bucket = System.getenv("amazon_s3_bucket") -String s3BasePath = System.getenv("amazon_s3_base_path") -String s3SessionToken = System.getenv("amazon_s3_session_token") - -if (!s3AccessKey && !s3SecretKey && !s3Bucket && !s3BasePath && !s3SessionToken) { - s3AccessKey = 's3_integration_test_access_key' - s3SecretKey = 's3_integration_test_secret_key' - s3Bucket = 'bucket_test' - s3BasePath = 'integration_test' +// We test against two repositories, one which uses the usual two-part "permanent" credentials and +// the other which uses three-part "temporary" or "session" credentials. + +String s3PermanentAccessKey = System.getenv("amazon_s3_access_key_permanent") +String s3PermanentSecretKey = System.getenv("amazon_s3_secret_key_permanent") +String s3PermanentBucket = System.getenv("amazon_s3_bucket_permanent") +String s3PermanentBasePath = System.getenv("amazon_s3_base_path_permanent") + +String s3TemporaryAccessKey = System.getenv("amazon_s3_access_key_temporary") +String s3TemporarySecretKey = System.getenv("amazon_s3_secret_key_temporary") +String s3TemporarySessionToken = System.getenv("amazon_s3_session_token_temporary") +String s3TemporaryBucket = System.getenv("amazon_s3_bucket_temporary") +String s3TemporaryBasePath = System.getenv("amazon_s3_base_path_temporary") + +// If all these variables are missing then we are testing against the internal fixture instead, which has the following +// credentials hard-coded in. + +if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath + && !s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3TemporaryBasePath && !s3TemporarySessionToken) { + + s3PermanentAccessKey = 's3_integration_test_permanent_access_key' + s3PermanentSecretKey = 's3_integration_test_permanent_secret_key' + s3PermanentBucket = 'permanent_bucket_test' + s3PermanentBasePath = 'integration_test' + + s3TemporaryAccessKey = 's3_integration_test_temporary_access_key' + s3TemporarySecretKey = 's3_integration_test_temporary_secret_key' + s3TemporaryBucket = 'temporary_bucket_test' + s3TemporaryBasePath = 'integration_test' + s3TemporarySessionToken = 's3_integration_test_temporary_session_token' + useFixture = true } @@ -58,12 +78,14 @@ task s3Fixture(type: AntFixture) { dependsOn compileTestJava env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }" executable = new File(project.runtimeJavaHome, 'bin/java') - args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir, s3Bucket + args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir } Map expansions = [ - 'bucket': s3Bucket, - 'base_path': s3BasePath + 'permanent_bucket': s3PermanentBucket, + 'permanent_base_path': s3PermanentBasePath, + 'temporary_bucket': s3TemporaryBucket, + 'temporary_base_path': s3TemporaryBasePath ] processTestResources { inputs.properties(expansions) @@ -71,16 +93,19 @@ processTestResources { } integTestCluster { - keystoreSetting 's3.client.integration_test.access_key', s3AccessKey - keystoreSetting 's3.client.integration_test.secret_key', s3SecretKey - if (s3SessionToken) { - keystoreSetting 's3.client.integration_test.session_token', s3SessionToken - } + keystoreSetting 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey + keystoreSetting 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey + + keystoreSetting 's3.client.integration_test_temporary.access_key', s3TemporaryAccessKey + keystoreSetting 's3.client.integration_test_temporary.secret_key', s3TemporarySecretKey + keystoreSetting 's3.client.integration_test_temporary.session_token', s3TemporarySessionToken if (useFixture) { + println "Using internal test service to test the repository-s3 plugin" dependsOn s3Fixture /* Use a closure on the string to delay evaluation until tests are executed */ - setting 's3.client.integration_test.endpoint', "http://${-> s3Fixture.addressAndPort}" + setting 's3.client.integration_test_permanent.endpoint', "http://${-> s3Fixture.addressAndPort}" + setting 's3.client.integration_test_temporary.endpoint', "http://${-> s3Fixture.addressAndPort}" } else { println "Using an external service to test the repository-s3 plugin" } diff --git a/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java b/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java index c8321e83d1390..23470ba1631f2 100644 --- a/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java +++ b/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java @@ -53,8 +53,8 @@ public class AmazonS3Fixture { public static void main(String[] args) throws Exception { - if (args == null || args.length != 2) { - throw new IllegalArgumentException("AmazonS3Fixture "); + if (args == null || args.length != 1) { + throw new IllegalArgumentException("AmazonS3Fixture "); } final InetSocketAddress socketAddress = new InetSocketAddress(InetAddress.getLoopbackAddress(), 0); @@ -72,7 +72,8 @@ public static void main(String[] args) throws Exception { // Emulates S3 final String storageUrl = "http://" + addressAndPort; final AmazonS3TestServer storageTestServer = new AmazonS3TestServer(storageUrl); - storageTestServer.createBucket(args[1]); + storageTestServer.createBucket("permanent_bucket_test"); + storageTestServer.createBucket("temporary_bucket_test"); httpServer.createContext("/", new ResponseHandler(storageTestServer)); httpServer.start(); diff --git a/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3TestServer.java b/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3TestServer.java index a3ea287b7f829..7706aa0436ac0 100644 --- a/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3TestServer.java +++ b/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3TestServer.java @@ -28,8 +28,11 @@ import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStreamReader; +import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Date; @@ -106,13 +109,38 @@ public Response handle(final String method, } final List authorizations = headers.get("Authorization"); - if (authorizations == null - || (authorizations.isEmpty() == false & authorizations.get(0).contains("s3_integration_test_access_key") == false)) { - return newError(requestId, RestStatus.FORBIDDEN, "AccessDenied", "Access Denied", ""); + if (authorizations == null || authorizations.size() != 1) { + return newError(requestId, RestStatus.FORBIDDEN, "AccessDenied", "No authorization", ""); + } + final String authorization = authorizations.get(0); + final String permittedBucket; + if (authorization.contains("s3_integration_test_permanent_access_key")) { + final List sessionTokens = headers.get("x-amz-security-token"); + if (sessionTokens != null) { + return newError(requestId, RestStatus.FORBIDDEN, "AccessDenied", "Unexpected session token", ""); + } + permittedBucket = "permanent_bucket_test"; + } else if (authorization.contains("s3_integration_test_temporary_access_key")) { + final List sessionTokens = headers.get("x-amz-security-token"); + if (sessionTokens == null || sessionTokens.size() != 1) { + return newError(requestId, RestStatus.FORBIDDEN, "AccessDenied", "No session token", ""); + } + final String sessionToken = sessionTokens.get(0); + if (sessionToken.equals("s3_integration_test_temporary_session_token") == false) { + return newError(requestId, RestStatus.FORBIDDEN, "AccessDenied", "Bad session token", ""); + } + permittedBucket = "temporary_bucket_test"; + } else { + return newError(requestId, RestStatus.FORBIDDEN, "AccessDenied", "Bad access key", ""); } final RequestHandler handler = handlers.retrieve(method + " " + path, params); if (handler != null) { + final String bucket = params.get("bucket"); + if (bucket != null && permittedBucket.equals(bucket) == false) { + // allow a null bucket to support bucket-free APIs like ListBuckets? + return newError(requestId, RestStatus.FORBIDDEN, "AccessDenied", "Bad bucket", ""); + } return handler.execute(params, headers, body, requestId); } else { return newInternalError(requestId, "No handler defined for request [method: " + method + ", path: " + path + "]"); diff --git a/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository.yml b/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository_permanent_credentials.yml similarity index 71% rename from plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository.yml rename to plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository_permanent_credentials.yml index 8b3daccf0a2d7..d7566af9a848c 100644 --- a/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository.yml +++ b/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository_permanent_credentials.yml @@ -1,17 +1,17 @@ # Integration tests for repository-s3 --- -"Snapshot/Restore with repository-s3": +"Snapshot/Restore with repository-s3 using permanent credentials": - # Register repository + # Register repository with permanent credentials - do: snapshot.create_repository: - repository: repository + repository: repository_permanent body: type: s3 settings: - bucket: ${bucket} - client: integration_test - base_path: ${base_path} + bucket: ${permanent_bucket} + client: integration_test_permanent + base_path: ${permanent_base_path} canned_acl: private storage_class: standard @@ -20,15 +20,16 @@ # Get repository - do: snapshot.get_repository: - repository: repository + repository: repository_permanent - - match: { repository.settings.bucket : ${bucket} } - - match: { repository.settings.client : "integration_test" } - - match: { repository.settings.base_path : ${base_path} } - - match: { repository.settings.canned_acl : "private" } - - match: { repository.settings.storage_class : "standard" } - - is_false: repository.settings.access_key - - is_false: repository.settings.secret_key + - match: { repository_permanent.settings.bucket : ${permanent_bucket} } + - match: { repository_permanent.settings.client : "integration_test_permanent" } + - match: { repository_permanent.settings.base_path : ${permanent_base_path} } + - match: { repository_permanent.settings.canned_acl : "private" } + - match: { repository_permanent.settings.storage_class : "standard" } + - is_false: repository_permanent.settings.access_key + - is_false: repository_permanent.settings.secret_key + - is_false: repository_permanent.settings.session_token # Index documents - do: @@ -60,7 +61,7 @@ # Create a first snapshot - do: snapshot.create: - repository: repository + repository: repository_permanent snapshot: snapshot-one wait_for_completion: true @@ -71,7 +72,7 @@ - do: snapshot.status: - repository: repository + repository: repository_permanent snapshot: snapshot-one - is_true: snapshots @@ -113,7 +114,7 @@ # Create a second snapshot - do: snapshot.create: - repository: repository + repository: repository_permanent snapshot: snapshot-two wait_for_completion: true @@ -123,7 +124,7 @@ - do: snapshot.get: - repository: repository + repository: repository_permanent snapshot: snapshot-one,snapshot-two - is_true: snapshots @@ -138,7 +139,7 @@ # Restore the second snapshot - do: snapshot.restore: - repository: repository + repository: repository_permanent snapshot: snapshot-two wait_for_completion: true @@ -156,7 +157,7 @@ # Restore the first snapshot - do: snapshot.restore: - repository: repository + repository: repository_permanent snapshot: snapshot-one wait_for_completion: true @@ -169,15 +170,15 @@ # Remove the snapshots - do: snapshot.delete: - repository: repository + repository: repository_permanent snapshot: snapshot-two - do: snapshot.delete: - repository: repository + repository: repository_permanent snapshot: snapshot-one # Remove our repository - do: snapshot.delete_repository: - repository: repository + repository: repository_permanent diff --git a/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_temporary_credentials.yml b/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_temporary_credentials.yml new file mode 100644 index 0000000000000..1e3671eedb7d7 --- /dev/null +++ b/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_temporary_credentials.yml @@ -0,0 +1,184 @@ +# Integration tests for repository-s3 +--- +"Snapshot/Restore with repository-s3 using temporary credentials": + + # Register repository with temporary credentials + - do: + snapshot.create_repository: + repository: repository_temporary + body: + type: s3 + settings: + bucket: ${temporary_bucket} + client: integration_test_temporary + base_path: ${temporary_base_path} + canned_acl: private + storage_class: standard + + - match: { acknowledged: true } + + # Get repository + - do: + snapshot.get_repository: + repository: repository_temporary + + - match: { repository_temporary.settings.bucket : ${temporary_bucket} } + - match: { repository_temporary.settings.client : "integration_test_temporary" } + - match: { repository_temporary.settings.base_path : ${temporary_base_path} } + - match: { repository_temporary.settings.canned_acl : "private" } + - match: { repository_temporary.settings.storage_class : "standard" } + - is_false: repository_temporary.settings.access_key + - is_false: repository_temporary.settings.secret_key + - is_false: repository_temporary.settings.session_token + + # Index documents + - do: + bulk: + refresh: true + body: + - index: + _index: docs + _type: doc + _id: 1 + - snapshot: one + - index: + _index: docs + _type: doc + _id: 2 + - snapshot: one + - index: + _index: docs + _type: doc + _id: 3 + - snapshot: one + + - do: + count: + index: docs + + - match: {count: 3} + + # Create a first snapshot + - do: + snapshot.create: + repository: repository_temporary + snapshot: snapshot-one + wait_for_completion: true + + - match: { snapshot.snapshot: snapshot-one } + - match: { snapshot.state : SUCCESS } + - match: { snapshot.include_global_state: true } + - match: { snapshot.shards.failed : 0 } + + - do: + snapshot.status: + repository: repository_temporary + snapshot: snapshot-one + + - is_true: snapshots + - match: { snapshots.0.snapshot: snapshot-one } + - match: { snapshots.0.state : SUCCESS } + + # Index more documents + - do: + bulk: + refresh: true + body: + - index: + _index: docs + _type: doc + _id: 4 + - snapshot: two + - index: + _index: docs + _type: doc + _id: 5 + - snapshot: two + - index: + _index: docs + _type: doc + _id: 6 + - snapshot: two + - index: + _index: docs + _type: doc + _id: 7 + - snapshot: two + + - do: + count: + index: docs + + - match: {count: 7} + + # Create a second snapshot + - do: + snapshot.create: + repository: repository_temporary + snapshot: snapshot-two + wait_for_completion: true + + - match: { snapshot.snapshot: snapshot-two } + - match: { snapshot.state : SUCCESS } + - match: { snapshot.shards.failed : 0 } + + - do: + snapshot.get: + repository: repository_temporary + snapshot: snapshot-one,snapshot-two + + - is_true: snapshots + - match: { snapshots.0.state : SUCCESS } + - match: { snapshots.1.state : SUCCESS } + + # Delete the index + - do: + indices.delete: + index: docs + + # Restore the second snapshot + - do: + snapshot.restore: + repository: repository_temporary + snapshot: snapshot-two + wait_for_completion: true + + - do: + count: + index: docs + + - match: {count: 7} + + # Delete the index again + - do: + indices.delete: + index: docs + + # Restore the first snapshot + - do: + snapshot.restore: + repository: repository_temporary + snapshot: snapshot-one + wait_for_completion: true + + - do: + count: + index: docs + + - match: {count: 3} + + # Remove the snapshots + - do: + snapshot.delete: + repository: repository_temporary + snapshot: snapshot-two + + - do: + snapshot.delete: + repository: repository_temporary + snapshot: snapshot-one + + # Remove our repository + - do: + snapshot.delete_repository: + repository: repository_temporary From d431f28e22fc3148ccea917c42e2b61b31839942 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 7 Jun 2018 11:45:45 +0100 Subject: [PATCH 07/21] Imports --- .../org/elasticsearch/repositories/s3/AmazonS3TestServer.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3TestServer.java b/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3TestServer.java index 7706aa0436ac0..0778ad7660c78 100644 --- a/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3TestServer.java +++ b/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3TestServer.java @@ -28,11 +28,8 @@ import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStreamReader; -import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Date; From 552e85221ea9bad74d6b8e66f66f7e65c4c52a35 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 25 Jun 2018 11:49:19 +0100 Subject: [PATCH 08/21] Add support for serving two buckets ... so that we can give them different accessibility. --- .../repositories/s3/AmazonS3Fixture.java | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java b/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java index 20e21675acb79..bc48911146dc1 100644 --- a/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java +++ b/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java @@ -53,12 +53,20 @@ public class AmazonS3Fixture extends AbstractHttpFixture { /** Request handlers for the requests made by the S3 client **/ private final PathTrie handlers; + private final String permanentBucketName; + private final String temporaryBucketName; + /** * Creates a {@link AmazonS3Fixture} + * @param permanentBucketName The name of the bucket that should be accessible using permanent (2-part) credentials. + * @param temporaryBucketName The name of the bucket that should be accessible using temporary (3-part) credentials. */ - private AmazonS3Fixture(final String workingDir, final String bucket) { + private AmazonS3Fixture(final String workingDir, final String permanentBucketName, final String temporaryBucketName) { super(workingDir); - this.buckets.put(bucket, new Bucket(bucket)); + this.permanentBucketName = permanentBucketName; + this.buckets.put(permanentBucketName, new Bucket(permanentBucketName)); + this.temporaryBucketName = temporaryBucketName; + this.buckets.put(temporaryBucketName, new Bucket(temporaryBucketName)); this.handlers = defaultHandlers(buckets); } @@ -77,11 +85,11 @@ protected Response handle(final Request request) throws IOException { } public static void main(final String[] args) throws Exception { - if (args == null || args.length != 2) { - throw new IllegalArgumentException("AmazonS3Fixture "); + if (args == null || args.length != 3) { + throw new IllegalArgumentException("AmazonS3Fixture "); } - final AmazonS3Fixture fixture = new AmazonS3Fixture(args[0], args[1]); + final AmazonS3Fixture fixture = new AmazonS3Fixture(args[0], args[1], args[2]); fixture.listen(); } From e583a0c78c97b7db144869a73e488e35130fbcdb Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 25 Jun 2018 11:59:42 +0100 Subject: [PATCH 09/21] Reinstate verification of credentials against the appropriate bucket --- .../repositories/s3/AmazonS3Fixture.java | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java b/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java index bc48911146dc1..89602202d9b6c 100644 --- a/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java +++ b/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java @@ -75,10 +75,30 @@ protected Response handle(final Request request) throws IOException { final RequestHandler handler = handlers.retrieve(request.getMethod() + " " + request.getPath(), request.getParameters()); if (handler != null) { final String authorization = request.getHeader("Authorization"); - if (authorization == null - || (authorization.length() > 0 && authorization.contains("s3_integration_test_access_key") == false)) { - return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Access Denied", ""); + + final String permittedBucket; + if (authorization.contains("s3_integration_test_permanent_access_key")) { + final String sessionToken = request.getHeader("x-amz-security-token"); + if (sessionToken != null) { + return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Unexpected session token", ""); + } + permittedBucket = permanentBucketName; + } else if (authorization.contains("s3_integration_test_temporary_access_key")) { + final String sessionToken = request.getHeader("x-amz-security-token"); + if (sessionToken.equals("s3_integration_test_temporary_session_token") == false) { + return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad session token", ""); + } + permittedBucket = temporaryBucketName; + } else { + return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad access key", ""); } + + final String bucket = request.getParam("bucket"); + if (bucket != null && permittedBucket.equals(bucket) == false) { + // allow a null bucket to support bucket-free APIs like ListBuckets? + return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad bucket", ""); + } + return handler.handle(request); } return null; From 87b6c584ef6b9c6fbb58ba55f11bdfceb1dbde30 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 25 Jun 2018 12:37:47 +0100 Subject: [PATCH 10/21] Update docs --- docs/plugins/discovery-ec2.asciidoc | 8 ++++++-- docs/plugins/repository-s3.asciidoc | 4 ++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/plugins/discovery-ec2.asciidoc b/docs/plugins/discovery-ec2.asciidoc index 2e2bc9cf268fa..5cdc568e16250 100644 --- a/docs/plugins/discovery-ec2.asciidoc +++ b/docs/plugins/discovery-ec2.asciidoc @@ -40,11 +40,15 @@ Those that must be stored in the keystore are marked as `Secure`. `access_key`:: - An s3 access key. The `secret_key` setting must also be specified. (Secure) + An ec2 access key. The `secret_key` setting must also be specified. (Secure) `secret_key`:: - An s3 secret key. The `access_key` setting must also be specified. (Secure) + An ec2 secret key. The `access_key` setting must also be specified. (Secure) + +`session_token`:: + An ec2 session token. The `access_key` and `secret_key` settings must also + be specified. (Secure) `endpoint`:: diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc index 6701d53c24047..0d73e35f18ec3 100644 --- a/docs/plugins/repository-s3.asciidoc +++ b/docs/plugins/repository-s3.asciidoc @@ -73,6 +73,10 @@ are marked as `Secure`. An s3 secret key. The `access_key` setting must also be specified. (Secure) +`session_token`:: + An s3 session token. The `access_key` and `secret_key` settings must also + be specified. (Secure) + `endpoint`:: The s3 service endpoint to connect to. This will be automatically From 7eabb5628f87522d6626b1a20481d3da802b3a6e Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 25 Jun 2018 12:38:48 +0100 Subject: [PATCH 11/21] Add discovery-ec2 session token setting --- .../org/elasticsearch/discovery/ec2/Ec2ClientSettings.java | 3 +++ .../org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java | 1 + 2 files changed, 4 insertions(+) diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2ClientSettings.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2ClientSettings.java index b42b0d546001a..d130191bcacd9 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2ClientSettings.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2ClientSettings.java @@ -42,6 +42,9 @@ final class Ec2ClientSettings { /** The secret key (ie password) for connecting to ec2. */ static final Setting SECRET_KEY_SETTING = SecureSetting.secureString("discovery.ec2.secret_key", null); + /** The session token for connecting to ec2. */ + static final Setting SESSION_TOKEN_SETTING = SecureSetting.secureString("discovery.ec2.session_token", null); + /** The host name of a proxy to connect to ec2 through. */ static final Setting PROXY_HOST_SETTING = Setting.simpleString("discovery.ec2.proxy.host", Property.NodeScope); diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java index 9fc32ea306c0e..bb757dc05adba 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java @@ -106,6 +106,7 @@ public List> getSettings() { // Register EC2 discovery settings: discovery.ec2 Ec2ClientSettings.ACCESS_KEY_SETTING, Ec2ClientSettings.SECRET_KEY_SETTING, + Ec2ClientSettings.SESSION_TOKEN_SETTING, Ec2ClientSettings.ENDPOINT_SETTING, Ec2ClientSettings.PROTOCOL_SETTING, Ec2ClientSettings.PROXY_HOST_SETTING, From 6915c4755712f96827994250d0da2a9e7284c198 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 25 Jun 2018 16:33:44 +0100 Subject: [PATCH 12/21] Use session token in discovery-ec2 if provided --- .../discovery/ec2/AwsEc2ServiceImpl.java | 10 ++-- .../discovery/ec2/Ec2ClientSettings.java | 59 ++++++++++++++----- .../discovery/ec2/AwsEc2ServiceImplTests.java | 55 ++++++++++++++--- .../common/settings/SettingsException.java | 4 ++ 4 files changed, 100 insertions(+), 28 deletions(-) diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java index 67902174630ea..a65500d9e2289 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java @@ -19,12 +19,9 @@ package org.elasticsearch.discovery.ec2; -import java.util.Random; -import java.util.concurrent.atomic.AtomicReference; - import com.amazonaws.ClientConfiguration; +import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; import com.amazonaws.http.IdleConnectionReaper; import com.amazonaws.internal.StaticCredentialsProvider; @@ -39,6 +36,9 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.LazyInitializable; +import java.util.Random; +import java.util.concurrent.atomic.AtomicReference; + class AwsEc2ServiceImpl extends AbstractComponent implements AwsEc2Service { public static final String EC2_METADATA_URL = "http://169.254.169.254/latest/meta-data/"; @@ -99,7 +99,7 @@ static ClientConfiguration buildConfiguration(Logger logger, Ec2ClientSettings c // pkg private for tests static AWSCredentialsProvider buildCredentials(Logger logger, Ec2ClientSettings clientSettings) { - final BasicAWSCredentials credentials = clientSettings.credentials; + final AWSCredentials credentials = clientSettings.credentials; if (credentials == null) { logger.debug("Using either environment variables, system properties or instance profile credentials"); return new DefaultAWSCredentialsProviderChain(); diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2ClientSettings.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2ClientSettings.java index d130191bcacd9..d76c9e820b8b1 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2ClientSettings.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2ClientSettings.java @@ -21,14 +21,20 @@ import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; +import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.BasicAWSCredentials; - +import com.amazonaws.auth.BasicSessionCredentials; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.SecureSetting; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Setting; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Setting.Property; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsException; import org.elasticsearch.common.unit.TimeValue; + import java.util.Locale; /** @@ -69,8 +75,12 @@ final class Ec2ClientSettings { static final Setting READ_TIMEOUT_SETTING = Setting.timeSetting("discovery.ec2.read_timeout", TimeValue.timeValueMillis(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT), Property.NodeScope); + private static final Logger logger = Loggers.getLogger(Ec2ClientSettings.class); + + private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(logger); + /** Credentials to authenticate with ec2. */ - final BasicAWSCredentials credentials; + final AWSCredentials credentials; /** * The ec2 endpoint the client should talk to, or empty string to use the @@ -99,7 +109,7 @@ final class Ec2ClientSettings { /** The read timeout for the ec2 client. */ final int readTimeoutMillis; - protected Ec2ClientSettings(BasicAWSCredentials credentials, String endpoint, Protocol protocol, String proxyHost, int proxyPort, + protected Ec2ClientSettings(AWSCredentials credentials, String endpoint, Protocol protocol, String proxyHost, int proxyPort, String proxyUsername, String proxyPassword, int readTimeoutMillis) { this.credentials = credentials; this.endpoint = endpoint; @@ -111,26 +121,45 @@ protected Ec2ClientSettings(BasicAWSCredentials credentials, String endpoint, Pr this.readTimeoutMillis = readTimeoutMillis; } - static BasicAWSCredentials loadCredentials(Settings settings) { - try (SecureString accessKey = ACCESS_KEY_SETTING.get(settings); - SecureString secretKey = SECRET_KEY_SETTING.get(settings);) { - if (accessKey.length() != 0) { - if (secretKey.length() != 0) { - return new BasicAWSCredentials(accessKey.toString(), secretKey.toString()); + static AWSCredentials loadCredentials(Settings settings) { + try (SecureString key = ACCESS_KEY_SETTING.get(settings); + SecureString secret = SECRET_KEY_SETTING.get(settings); + SecureString sessionToken = SESSION_TOKEN_SETTING.get(settings)) { + if (key.length() == 0 && secret.length() == 0) { + if (sessionToken.length() > 0) { + throw new SettingsException("Setting [{}] is set but [{}] and [{}] are not", + SESSION_TOKEN_SETTING.getKey(), ACCESS_KEY_SETTING.getKey(), SECRET_KEY_SETTING.getKey()); + } + + logger.debug("Using either environment variables, system properties or instance profile credentials"); + return null; + } else { + if (key.length() == 0) { + DEPRECATION_LOGGER.deprecated("Setting [{}] is set but [{}] is not, which will be unsupported in future", + SECRET_KEY_SETTING.getKey(), ACCESS_KEY_SETTING.getKey()); + } + if (secret.length() == 0) { + DEPRECATION_LOGGER.deprecated("Setting [{}] is set but [{}] is not, which will be unsupported in future", + ACCESS_KEY_SETTING.getKey(), SECRET_KEY_SETTING.getKey()); + } + + final AWSCredentials credentials; + if (sessionToken.length() == 0) { + logger.debug("Using basic key/secret credentials"); + credentials = new BasicAWSCredentials(key.toString(), secret.toString()); } else { - throw new IllegalArgumentException("Missing secret key for ec2 client."); + logger.debug("Using basic session credentials"); + credentials = new BasicSessionCredentials(key.toString(), secret.toString(), sessionToken.toString()); } - } else if (secretKey.length() != 0) { - throw new IllegalArgumentException("Missing access key for ec2 client."); + return credentials; } - return null; } } // pkg private for tests /** Parse settings for a single client. */ static Ec2ClientSettings getClientSettings(Settings settings) { - final BasicAWSCredentials credentials = loadCredentials(settings); + final AWSCredentials credentials = loadCredentials(settings); try (SecureString proxyUsername = PROXY_USERNAME_SETTING.get(settings); SecureString proxyPassword = PROXY_PASSWORD_SETTING.get(settings)) { return new Ec2ClientSettings( diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImplTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImplTests.java index a13fe47a632ae..148e58d7b3c06 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImplTests.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImplTests.java @@ -23,10 +23,11 @@ import com.amazonaws.Protocol; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.BasicSessionCredentials; import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.discovery.ec2.AwsEc2ServiceImpl; +import org.elasticsearch.common.settings.SettingsException; import org.elasticsearch.test.ESTestCase; import static org.hamcrest.Matchers.instanceOf; @@ -44,15 +45,53 @@ public void testAWSCredentialsWithElasticsearchAwsSettings() { final MockSecureSettings secureSettings = new MockSecureSettings(); secureSettings.setString("discovery.ec2.access_key", "aws_key"); secureSettings.setString("discovery.ec2.secret_key", "aws_secret"); - final Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); - launchAWSCredentialsWithElasticsearchSettingsTest(settings, "aws_key", "aws_secret"); + final AWSCredentials credentials = AwsEc2ServiceImpl.buildCredentials(logger, + Ec2ClientSettings.getClientSettings(Settings.builder().setSecureSettings(secureSettings).build())).getCredentials(); + assertThat(credentials.getAWSAccessKeyId(), is("aws_key")); + assertThat(credentials.getAWSSecretKey(), is("aws_secret")); } - protected void launchAWSCredentialsWithElasticsearchSettingsTest(Settings settings, String expectedKey, String expectedSecret) { - final AWSCredentials credentials = AwsEc2ServiceImpl.buildCredentials(logger, Ec2ClientSettings.getClientSettings(settings)) - .getCredentials(); - assertThat(credentials.getAWSAccessKeyId(), is(expectedKey)); - assertThat(credentials.getAWSSecretKey(), is(expectedSecret)); + public void testAWSSessionCredentialsWithElasticsearchAwsSettings() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("discovery.ec2.access_key", "aws_key"); + secureSettings.setString("discovery.ec2.secret_key", "aws_secret"); + secureSettings.setString("discovery.ec2.session_token", "aws_session_token"); + final BasicSessionCredentials credentials = (BasicSessionCredentials) AwsEc2ServiceImpl.buildCredentials(logger, + Ec2ClientSettings.getClientSettings(Settings.builder().setSecureSettings(secureSettings).build())).getCredentials(); + assertThat(credentials.getAWSAccessKeyId(), is("aws_key")); + assertThat(credentials.getAWSSecretKey(), is("aws_secret")); + assertThat(credentials.getSessionToken(), is("aws_session_token")); + } + + public void testDeprecationOfLoneAccessKey() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("discovery.ec2.access_key", "aws_key"); + final AWSCredentials credentials = AwsEc2ServiceImpl.buildCredentials(logger, + Ec2ClientSettings.getClientSettings(Settings.builder().setSecureSettings(secureSettings).build())).getCredentials(); + assertThat(credentials.getAWSAccessKeyId(), is("aws_key")); + assertThat(credentials.getAWSSecretKey(), is("")); + assertSettingDeprecationsAndWarnings(new String[]{}, + "Setting [discovery.ec2.access_key] is set but [discovery.ec2.secret_key] is not, which will be unsupported in future"); + } + + public void testDeprecationOfLoneSecretKey() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("discovery.ec2.secret_key", "aws_secret"); + final AWSCredentials credentials = AwsEc2ServiceImpl.buildCredentials(logger, + Ec2ClientSettings.getClientSettings(Settings.builder().setSecureSettings(secureSettings).build())).getCredentials(); + assertThat(credentials.getAWSAccessKeyId(), is("")); + assertThat(credentials.getAWSSecretKey(), is("aws_secret")); + assertSettingDeprecationsAndWarnings(new String[]{}, + "Setting [discovery.ec2.secret_key] is set but [discovery.ec2.access_key] is not, which will be unsupported in future"); + } + + public void testRejectionOfLoneSessionToken() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("discovery.ec2.session_token", "aws_session_token"); + SettingsException e = expectThrows(SettingsException.class, () -> AwsEc2ServiceImpl.buildCredentials(logger, + Ec2ClientSettings.getClientSettings(Settings.builder().setSecureSettings(secureSettings).build()))); + assertThat(e.getMessage(), is( + "Setting [discovery.ec2.session_token] is set but [discovery.ec2.access_key] and [discovery.ec2.secret_key] are not")); } public void testAWSDefaultConfiguration() { diff --git a/server/src/main/java/org/elasticsearch/common/settings/SettingsException.java b/server/src/main/java/org/elasticsearch/common/settings/SettingsException.java index f7d4843c1c03e..ad5f56d7fc0f3 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/SettingsException.java +++ b/server/src/main/java/org/elasticsearch/common/settings/SettingsException.java @@ -42,4 +42,8 @@ public SettingsException(String message, Throwable cause) { public SettingsException(StreamInput in) throws IOException { super(in); } + + public SettingsException(String msg, Object... args) { + super(msg, args); + } } From 1475a263a47c1e7e633faba5b1bb48cdd806fca4 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 25 Jun 2018 16:40:20 +0100 Subject: [PATCH 13/21] Add assertions to Ec2DiscoveryPluginTests too --- .../ec2/Ec2DiscoveryPluginTests.java | 68 ++++++++++++++----- 1 file changed, 52 insertions(+), 16 deletions(-) diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java index 6001ab56d5042..0db9e9dc8235b 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java @@ -25,9 +25,14 @@ import java.nio.file.Path; import java.util.Arrays; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.auth.BasicSessionCredentials; +import org.elasticsearch.cluster.ClusterStateTaskConfig.Basic; import org.elasticsearch.discovery.ec2.AwsEc2Service; import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.Settings; @@ -106,6 +111,10 @@ public void testClientSettingsReInit() throws IOException { final MockSecureSettings mockSecure1 = new MockSecureSettings(); mockSecure1.setString(Ec2ClientSettings.ACCESS_KEY_SETTING.getKey(), "ec2_access_1"); mockSecure1.setString(Ec2ClientSettings.SECRET_KEY_SETTING.getKey(), "ec2_secret_1"); + final boolean mockSecure1HasSessionToken = randomBoolean(); + if (mockSecure1HasSessionToken) { + mockSecure1.setString(Ec2ClientSettings.SESSION_TOKEN_SETTING.getKey(), "ec2_session_token_1"); + } mockSecure1.setString(Ec2ClientSettings.PROXY_USERNAME_SETTING.getKey(), "proxy_username_1"); mockSecure1.setString(Ec2ClientSettings.PROXY_PASSWORD_SETTING.getKey(), "proxy_password_1"); final Settings settings1 = Settings.builder() @@ -117,6 +126,10 @@ public void testClientSettingsReInit() throws IOException { final MockSecureSettings mockSecure2 = new MockSecureSettings(); mockSecure2.setString(Ec2ClientSettings.ACCESS_KEY_SETTING.getKey(), "ec2_access_2"); mockSecure2.setString(Ec2ClientSettings.SECRET_KEY_SETTING.getKey(), "ec2_secret_2"); + final boolean mockSecure2HasSessionToken = randomBoolean(); + if (mockSecure2HasSessionToken) { + mockSecure2.setString(Ec2ClientSettings.SESSION_TOKEN_SETTING.getKey(), "ec2_session_token_2"); + } mockSecure2.setString(Ec2ClientSettings.PROXY_USERNAME_SETTING.getKey(), "proxy_username_2"); mockSecure2.setString(Ec2ClientSettings.PROXY_PASSWORD_SETTING.getKey(), "proxy_password_2"); final Settings settings2 = Settings.builder() @@ -127,27 +140,50 @@ public void testClientSettingsReInit() throws IOException { .build(); try (Ec2DiscoveryPluginMock plugin = new Ec2DiscoveryPluginMock(settings1)) { try (AmazonEc2Reference clientReference = plugin.ec2Service.client()) { - assertThat(((AmazonEC2Mock) clientReference.client()).credentials.getCredentials().getAWSAccessKeyId(), is("ec2_access_1")); - assertThat(((AmazonEC2Mock) clientReference.client()).credentials.getCredentials().getAWSSecretKey(), is("ec2_secret_1")); - assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyUsername(), is("proxy_username_1")); - assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPassword(), is("proxy_password_1")); - assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyHost(), is("proxy_host_1")); - assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPort(), is(881)); - assertThat(((AmazonEC2Mock) clientReference.client()).endpoint, is("ec2_endpoint_1")); + { + final AWSCredentials credentials = ((AmazonEC2Mock) clientReference.client()).credentials.getCredentials(); + assertThat(credentials.getAWSAccessKeyId(), is("ec2_access_1")); + assertThat(credentials.getAWSSecretKey(), is("ec2_secret_1")); + if (mockSecure1HasSessionToken) { + assertThat(credentials, instanceOf(BasicSessionCredentials.class)); + assertThat(((BasicSessionCredentials)credentials).getSessionToken(), is("ec2_session_token_1")); + } else { + assertThat(credentials, instanceOf(BasicAWSCredentials.class)); + } + assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyUsername(), is("proxy_username_1")); + assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPassword(), is("proxy_password_1")); + assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyHost(), is("proxy_host_1")); + assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPort(), is(881)); + assertThat(((AmazonEC2Mock) clientReference.client()).endpoint, is("ec2_endpoint_1")); + } // reload secure settings2 plugin.reload(settings2); // client is not released, it is still using the old settings - assertThat(((AmazonEC2Mock) clientReference.client()).credentials.getCredentials().getAWSAccessKeyId(), is("ec2_access_1")); - assertThat(((AmazonEC2Mock) clientReference.client()).credentials.getCredentials().getAWSSecretKey(), is("ec2_secret_1")); - assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyUsername(), is("proxy_username_1")); - assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPassword(), is("proxy_password_1")); - assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyHost(), is("proxy_host_1")); - assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPort(), is(881)); - assertThat(((AmazonEC2Mock) clientReference.client()).endpoint, is("ec2_endpoint_1")); + { + final AWSCredentials credentials = ((AmazonEC2Mock) clientReference.client()).credentials.getCredentials(); + if (mockSecure1HasSessionToken) { + assertThat(credentials, instanceOf(BasicSessionCredentials.class)); + assertThat(((BasicSessionCredentials)credentials).getSessionToken(), is("ec2_session_token_1")); + } else { + assertThat(credentials, instanceOf(BasicAWSCredentials.class)); + } + assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyUsername(), is("proxy_username_1")); + assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPassword(), is("proxy_password_1")); + assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyHost(), is("proxy_host_1")); + assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPort(), is(881)); + assertThat(((AmazonEC2Mock) clientReference.client()).endpoint, is("ec2_endpoint_1")); + } } try (AmazonEc2Reference clientReference = plugin.ec2Service.client()) { - assertThat(((AmazonEC2Mock) clientReference.client()).credentials.getCredentials().getAWSAccessKeyId(), is("ec2_access_2")); - assertThat(((AmazonEC2Mock) clientReference.client()).credentials.getCredentials().getAWSSecretKey(), is("ec2_secret_2")); + final AWSCredentials credentials = ((AmazonEC2Mock) clientReference.client()).credentials.getCredentials(); + assertThat(credentials.getAWSAccessKeyId(), is("ec2_access_2")); + assertThat(credentials.getAWSSecretKey(), is("ec2_secret_2")); + if (mockSecure2HasSessionToken) { + assertThat(credentials, instanceOf(BasicSessionCredentials.class)); + assertThat(((BasicSessionCredentials)credentials).getSessionToken(), is("ec2_session_token_2")); + } else { + assertThat(credentials, instanceOf(BasicAWSCredentials.class)); + } assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyUsername(), is("proxy_username_2")); assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPassword(), is("proxy_password_2")); assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyHost(), is("proxy_host_2")); From 439b76314c59901e1a62112206253088092f39d3 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 25 Jun 2018 16:51:48 +0100 Subject: [PATCH 14/21] Add support for session token to repository-s3 plugin --- .../repositories/s3/InternalAwsS3Service.java | 2 +- .../repositories/s3/S3ClientSettings.java | 34 +++-- .../repositories/s3/S3RepositoryPlugin.java | 1 + .../s3/S3ClientSettingsTests.java | 123 ++++++++++++++++++ 4 files changed, 150 insertions(+), 10 deletions(-) create mode 100644 plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java index a54320f1fbd19..b372e9e0154f4 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java @@ -135,7 +135,7 @@ static ClientConfiguration buildConfiguration(S3ClientSettings clientSettings) { // pkg private for tests static AWSCredentialsProvider buildCredentials(Logger logger, S3ClientSettings clientSettings) { - final BasicAWSCredentials credentials = clientSettings.credentials; + final AWSCredentials credentials = clientSettings.credentials; if (credentials == null) { logger.debug("Using instance profile credentials"); return new PrivilegedInstanceProfileCredentialsProvider(); diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java index ef6088fe154bf..795304541be35 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java @@ -26,8 +26,10 @@ import java.util.Set; import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; +import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.auth.BasicSessionCredentials; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.settings.SecureSetting; import org.elasticsearch.common.settings.SecureString; @@ -52,6 +54,10 @@ final class S3ClientSettings { static final Setting.AffixSetting SECRET_KEY_SETTING = Setting.affixKeySetting(PREFIX, "secret_key", key -> SecureSetting.secureString(key, null)); + /** The secret key (ie password) for connecting to s3. */ + static final Setting.AffixSetting SESSION_TOKEN_SETTING = Setting.affixKeySetting(PREFIX, "session_token", + key -> SecureSetting.secureString(key, null)); + /** An override for the s3 endpoint to connect to. */ static final Setting.AffixSetting ENDPOINT_SETTING = Setting.affixKeySetting(PREFIX, "endpoint", key -> new Setting<>(key, "", s -> s.toLowerCase(Locale.ROOT), Property.NodeScope)); @@ -89,7 +95,7 @@ final class S3ClientSettings { key -> Setting.boolSetting(key, ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope)); /** Credentials to authenticate with s3. */ - final BasicAWSCredentials credentials; + final AWSCredentials credentials; /** The s3 endpoint the client should talk to, or empty string to use the default. */ final String endpoint; @@ -120,7 +126,7 @@ final class S3ClientSettings { /** Whether the s3 client should use an exponential backoff retry policy. */ final boolean throttleRetries; - protected S3ClientSettings(BasicAWSCredentials credentials, String endpoint, Protocol protocol, + protected S3ClientSettings(AWSCredentials credentials, String endpoint, Protocol protocol, String proxyHost, int proxyPort, String proxyUsername, String proxyPassword, int readTimeoutMillis, int maxRetries, boolean throttleRetries) { this.credentials = credentials; @@ -190,26 +196,36 @@ static BasicAWSCredentials loadDeprecatedCredentials(Settings repositorySettings } } - static BasicAWSCredentials loadCredentials(Settings settings, String clientName) { + static AWSCredentials loadCredentials(Settings settings, String clientName) { try (SecureString accessKey = getConfigValue(settings, clientName, ACCESS_KEY_SETTING); - SecureString secretKey = getConfigValue(settings, clientName, SECRET_KEY_SETTING);) { + SecureString secretKey = getConfigValue(settings, clientName, SECRET_KEY_SETTING); + SecureString sessionToken = getConfigValue(settings, clientName, SESSION_TOKEN_SETTING)) { if (accessKey.length() != 0) { if (secretKey.length() != 0) { - return new BasicAWSCredentials(accessKey.toString(), secretKey.toString()); + if (sessionToken.length() != 0) { + return new BasicSessionCredentials(accessKey.toString(), secretKey.toString(), sessionToken.toString()); + } else { + return new BasicAWSCredentials(accessKey.toString(), secretKey.toString()); + } } else { throw new IllegalArgumentException("Missing secret key for s3 client [" + clientName + "]"); } - } else if (secretKey.length() != 0) { - throw new IllegalArgumentException("Missing access key for s3 client [" + clientName + "]"); + } else { + if (secretKey.length() != 0) { + throw new IllegalArgumentException("Missing access key for s3 client [" + clientName + "]"); + } + if (sessionToken.length() != 0) { + throw new IllegalArgumentException("Missing access key and secret key for s3 client [" + clientName + "]"); + } + return null; } - return null; } } // pkg private for tests /** Parse settings for a single client. */ static S3ClientSettings getClientSettings(Settings settings, String clientName) { - final BasicAWSCredentials credentials = S3ClientSettings.loadCredentials(settings, clientName); + final AWSCredentials credentials = S3ClientSettings.loadCredentials(settings, clientName); try (SecureString proxyUsername = getConfigValue(settings, clientName, PROXY_USERNAME_SETTING); SecureString proxyPassword = getConfigValue(settings, clientName, PROXY_PASSWORD_SETTING)) { return new S3ClientSettings( diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java index 93561c94d2b9a..a28fff79cd848 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java @@ -95,6 +95,7 @@ public List> getSettings() { // named s3 client configuration settings S3ClientSettings.ACCESS_KEY_SETTING, S3ClientSettings.SECRET_KEY_SETTING, + S3ClientSettings.SESSION_TOKEN_SETTING, S3ClientSettings.ENDPOINT_SETTING, S3ClientSettings.PROTOCOL_SETTING, S3ClientSettings.PROXY_HOST_SETTING, diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java new file mode 100644 index 0000000000000..e629f43f8a3d3 --- /dev/null +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java @@ -0,0 +1,123 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.repositories.s3; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.auth.BasicSessionCredentials; +import org.elasticsearch.common.settings.MockSecureSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; + +import java.util.Map; + +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.isEmptyString; +import static org.hamcrest.Matchers.nullValue; + +public class S3ClientSettingsTests extends ESTestCase { + public void testThereIsADefaultClientByDefault() { + final Map settings = S3ClientSettings.load(Settings.EMPTY); + assertThat(settings.keySet(), contains("default")); + + final S3ClientSettings defaultSettings = settings.get("default"); + assertThat(defaultSettings.credentials, nullValue()); + assertThat(defaultSettings.endpoint, isEmptyString()); + assertThat(defaultSettings.protocol, is(Protocol.HTTPS)); + assertThat(defaultSettings.proxyHost, isEmptyString()); + assertThat(defaultSettings.proxyPort, is(80)); + assertThat(defaultSettings.proxyUsername, isEmptyString()); + assertThat(defaultSettings.proxyPassword, isEmptyString()); + assertThat(defaultSettings.readTimeoutMillis, is(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT)); + assertThat(defaultSettings.maxRetries, is(ClientConfiguration.DEFAULT_RETRY_POLICY.getMaxErrorRetry())); + assertThat(defaultSettings.throttleRetries, is(ClientConfiguration.DEFAULT_THROTTLE_RETRIES)); + } + + public void testDefaultClientSettingsCanBeSet() { + final Map settings = S3ClientSettings.load(Settings.builder() + .put("s3.client.default.max_retries", 10).build()); + assertThat(settings.keySet(), contains("default")); + + final S3ClientSettings defaultSettings = settings.get("default"); + assertThat(defaultSettings.maxRetries, is(10)); + } + + public void testNondefaultClientCreatedBySettingItsSettings() { + final Map settings = S3ClientSettings.load(Settings.builder() + .put("s3.client.another_client.max_retries", 10).build()); + assertThat(settings.keySet(), contains("default", "another_client")); + + final S3ClientSettings defaultSettings = settings.get("default"); + assertThat(defaultSettings.maxRetries, is(ClientConfiguration.DEFAULT_RETRY_POLICY.getMaxErrorRetry())); + + final S3ClientSettings anotherClientSettings = settings.get("another_client"); + assertThat(anotherClientSettings.maxRetries, is(10)); + } + + public void testRejectionOfLoneAccessKey() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("s3.client.default.access_key", "aws_key"); + final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build())); + assertThat(e.getMessage(), is("Missing secret key for s3 client [default]")); + } + + public void testRejectionOfLoneSecretKey() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("s3.client.default.secret_key", "aws_key"); + final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build())); + assertThat(e.getMessage(), is("Missing access key for s3 client [default]")); + } + + public void testRejectionOfLoneSessionToken() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("s3.client.default.session_token", "aws_key"); + final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build())); + assertThat(e.getMessage(), is("Missing access key and secret key for s3 client [default]")); + } + + public void testCredentialsTypeWithAccessKeyAndSecretKey() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("s3.client.default.access_key", "access_key"); + secureSettings.setString("s3.client.default.secret_key", "secret_key"); + final Map settings = S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()); + final S3ClientSettings defaultSettings = settings.get("default"); + BasicAWSCredentials credentials = (BasicAWSCredentials) defaultSettings.credentials; + assertThat(credentials.getAWSAccessKeyId(), is("access_key")); + assertThat(credentials.getAWSSecretKey(), is("secret_key")); + } + + public void testCredentialsTypeWithAccessKeyAndSecretKeyAndSessionToken() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("s3.client.default.access_key", "access_key"); + secureSettings.setString("s3.client.default.secret_key", "secret_key"); + secureSettings.setString("s3.client.default.session_token", "session_token"); + final Map settings = S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()); + final S3ClientSettings defaultSettings = settings.get("default"); + BasicSessionCredentials credentials = (BasicSessionCredentials) defaultSettings.credentials; + assertThat(credentials.getAWSAccessKeyId(), is("access_key")); + assertThat(credentials.getAWSSecretKey(), is("secret_key")); + assertThat(credentials.getSessionToken(), is("session_token")); + } +} From 2a93e9490960fa9237cc06294fff5829c3d5e881 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 25 Jun 2018 17:49:07 +0100 Subject: [PATCH 15/21] Extend QA tests to test both permanent and temporary credentials --- .../repository-s3/qa/amazon-s3/build.gradle | 68 +++++-- .../repositories/s3/AmazonS3Fixture.java | 15 +- ...> 10_repository_permanent_credentials.yml} | 47 ++--- .../20_repository_temporary_credentials.yml | 184 ++++++++++++++++++ .../repositories/s3/InternalAwsS3Service.java | 3 +- 5 files changed, 269 insertions(+), 48 deletions(-) rename plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/{10_repository.yml => 10_repository_permanent_credentials.yml} (71%) create mode 100644 plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_temporary_credentials.yml diff --git a/plugins/repository-s3/qa/amazon-s3/build.gradle b/plugins/repository-s3/qa/amazon-s3/build.gradle index dbbffdebded47..b6cc4a6de310d 100644 --- a/plugins/repository-s3/qa/amazon-s3/build.gradle +++ b/plugins/repository-s3/qa/amazon-s3/build.gradle @@ -31,47 +31,81 @@ integTestCluster { plugin ':plugins:repository-s3' } +forbiddenApisTest { + // we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage + bundledSignatures -= 'jdk-non-portable' + bundledSignatures += 'jdk-internal' +} + boolean useFixture = false -String s3AccessKey = System.getenv("amazon_s3_access_key") -String s3SecretKey = System.getenv("amazon_s3_secret_key") -String s3Bucket = System.getenv("amazon_s3_bucket") -String s3BasePath = System.getenv("amazon_s3_base_path") +// We test against two repositories, one which uses the usual two-part "permanent" credentials and +// the other which uses three-part "temporary" or "session" credentials. + +String s3PermanentAccessKey = System.getenv("amazon_s3_access_key") +String s3PermanentSecretKey = System.getenv("amazon_s3_secret_key") +String s3PermanentBucket = System.getenv("amazon_s3_bucket") +String s3PermanentBasePath = System.getenv("amazon_s3_base_path") + +String s3TemporaryAccessKey = System.getenv("amazon_s3_access_key_temporary") +String s3TemporarySecretKey = System.getenv("amazon_s3_secret_key_temporary") +String s3TemporarySessionToken = System.getenv("amazon_s3_session_token_temporary") +String s3TemporaryBucket = System.getenv("amazon_s3_bucket_temporary") +String s3TemporaryBasePath = System.getenv("amazon_s3_base_path_temporary") + +// If all these variables are missing then we are testing against the internal fixture instead, which has the following +// credentials hard-coded in. + +if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath + && !s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3TemporaryBasePath && !s3TemporarySessionToken) { + + s3PermanentAccessKey = 's3_integration_test_permanent_access_key' + s3PermanentSecretKey = 's3_integration_test_permanent_secret_key' + s3PermanentBucket = 'permanent_bucket_test' + s3PermanentBasePath = 'integration_test' + + s3TemporaryAccessKey = 's3_integration_test_temporary_access_key' + s3TemporarySecretKey = 's3_integration_test_temporary_secret_key' + s3TemporaryBucket = 'temporary_bucket_test' + s3TemporaryBasePath = 'integration_test' + s3TemporarySessionToken = 's3_integration_test_temporary_session_token' -if (!s3AccessKey && !s3SecretKey && !s3Bucket && !s3BasePath) { - s3AccessKey = 's3_integration_test_access_key' - s3SecretKey = 's3_integration_test_secret_key' - s3Bucket = 'bucket_test' - s3BasePath = 'integration_test' useFixture = true } /** A task to start the AmazonS3Fixture which emulates a S3 service **/ task s3Fixture(type: AntFixture) { - dependsOn testClasses + dependsOn compileTestJava env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }" executable = new File(project.runtimeJavaHome, 'bin/java') - args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir, s3Bucket + args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir, s3PermanentBucket, s3TemporaryBucket } Map expansions = [ - 'bucket': s3Bucket, - 'base_path': s3BasePath + 'permanent_bucket': s3PermanentBucket, + 'permanent_base_path': s3PermanentBasePath, + 'temporary_bucket': s3TemporaryBucket, + 'temporary_base_path': s3TemporaryBasePath ] - processTestResources { inputs.properties(expansions) MavenFilteringHack.filter(it, expansions) } integTestCluster { - keystoreSetting 's3.client.integration_test.access_key', s3AccessKey - keystoreSetting 's3.client.integration_test.secret_key', s3SecretKey + keystoreSetting 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey + keystoreSetting 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey + + keystoreSetting 's3.client.integration_test_temporary.access_key', s3TemporaryAccessKey + keystoreSetting 's3.client.integration_test_temporary.secret_key', s3TemporarySecretKey + keystoreSetting 's3.client.integration_test_temporary.session_token', s3TemporarySessionToken if (useFixture) { + println "Using internal test service to test the repository-s3 plugin" dependsOn s3Fixture /* Use a closure on the string to delay evaluation until tests are executed */ - setting 's3.client.integration_test.endpoint', "http://${-> s3Fixture.addressAndPort}" + setting 's3.client.integration_test_permanent.endpoint', "http://${-> s3Fixture.addressAndPort}" + setting 's3.client.integration_test_temporary.endpoint', "http://${-> s3Fixture.addressAndPort}" } else { println "Using an external service to test the repository-s3 plugin" } diff --git a/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java b/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java index 20e21675acb79..26aaf5b3661d9 100644 --- a/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java +++ b/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java @@ -56,9 +56,10 @@ public class AmazonS3Fixture extends AbstractHttpFixture { /** * Creates a {@link AmazonS3Fixture} */ - private AmazonS3Fixture(final String workingDir, final String bucket) { + private AmazonS3Fixture(final String workingDir, final String permanentBucketName, final String temporaryBucketName) { super(workingDir); - this.buckets.put(bucket, new Bucket(bucket)); + this.buckets.put(permanentBucketName, new Bucket(permanentBucketName)); + this.buckets.put(temporaryBucketName, new Bucket(temporaryBucketName)); this.handlers = defaultHandlers(buckets); } @@ -68,7 +69,8 @@ protected Response handle(final Request request) throws IOException { if (handler != null) { final String authorization = request.getHeader("Authorization"); if (authorization == null - || (authorization.length() > 0 && authorization.contains("s3_integration_test_access_key") == false)) { + || (authorization.length() > 0 && authorization.contains("s3_integration_test_permanent_access_key") == false + && authorization.contains("s3_integration_test_temporary_access_key") == false)) { return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Access Denied", ""); } return handler.handle(request); @@ -77,11 +79,12 @@ protected Response handle(final Request request) throws IOException { } public static void main(final String[] args) throws Exception { - if (args == null || args.length != 2) { - throw new IllegalArgumentException("AmazonS3Fixture "); + if (args == null || args.length != 3) { + throw new IllegalArgumentException( + "AmazonS3Fixture "); } - final AmazonS3Fixture fixture = new AmazonS3Fixture(args[0], args[1]); + final AmazonS3Fixture fixture = new AmazonS3Fixture(args[0], args[1], args[2]); fixture.listen(); } diff --git a/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository.yml b/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository_permanent_credentials.yml similarity index 71% rename from plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository.yml rename to plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository_permanent_credentials.yml index 8b3daccf0a2d7..d7566af9a848c 100644 --- a/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository.yml +++ b/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository_permanent_credentials.yml @@ -1,17 +1,17 @@ # Integration tests for repository-s3 --- -"Snapshot/Restore with repository-s3": +"Snapshot/Restore with repository-s3 using permanent credentials": - # Register repository + # Register repository with permanent credentials - do: snapshot.create_repository: - repository: repository + repository: repository_permanent body: type: s3 settings: - bucket: ${bucket} - client: integration_test - base_path: ${base_path} + bucket: ${permanent_bucket} + client: integration_test_permanent + base_path: ${permanent_base_path} canned_acl: private storage_class: standard @@ -20,15 +20,16 @@ # Get repository - do: snapshot.get_repository: - repository: repository + repository: repository_permanent - - match: { repository.settings.bucket : ${bucket} } - - match: { repository.settings.client : "integration_test" } - - match: { repository.settings.base_path : ${base_path} } - - match: { repository.settings.canned_acl : "private" } - - match: { repository.settings.storage_class : "standard" } - - is_false: repository.settings.access_key - - is_false: repository.settings.secret_key + - match: { repository_permanent.settings.bucket : ${permanent_bucket} } + - match: { repository_permanent.settings.client : "integration_test_permanent" } + - match: { repository_permanent.settings.base_path : ${permanent_base_path} } + - match: { repository_permanent.settings.canned_acl : "private" } + - match: { repository_permanent.settings.storage_class : "standard" } + - is_false: repository_permanent.settings.access_key + - is_false: repository_permanent.settings.secret_key + - is_false: repository_permanent.settings.session_token # Index documents - do: @@ -60,7 +61,7 @@ # Create a first snapshot - do: snapshot.create: - repository: repository + repository: repository_permanent snapshot: snapshot-one wait_for_completion: true @@ -71,7 +72,7 @@ - do: snapshot.status: - repository: repository + repository: repository_permanent snapshot: snapshot-one - is_true: snapshots @@ -113,7 +114,7 @@ # Create a second snapshot - do: snapshot.create: - repository: repository + repository: repository_permanent snapshot: snapshot-two wait_for_completion: true @@ -123,7 +124,7 @@ - do: snapshot.get: - repository: repository + repository: repository_permanent snapshot: snapshot-one,snapshot-two - is_true: snapshots @@ -138,7 +139,7 @@ # Restore the second snapshot - do: snapshot.restore: - repository: repository + repository: repository_permanent snapshot: snapshot-two wait_for_completion: true @@ -156,7 +157,7 @@ # Restore the first snapshot - do: snapshot.restore: - repository: repository + repository: repository_permanent snapshot: snapshot-one wait_for_completion: true @@ -169,15 +170,15 @@ # Remove the snapshots - do: snapshot.delete: - repository: repository + repository: repository_permanent snapshot: snapshot-two - do: snapshot.delete: - repository: repository + repository: repository_permanent snapshot: snapshot-one # Remove our repository - do: snapshot.delete_repository: - repository: repository + repository: repository_permanent diff --git a/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_temporary_credentials.yml b/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_temporary_credentials.yml new file mode 100644 index 0000000000000..1e3671eedb7d7 --- /dev/null +++ b/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_temporary_credentials.yml @@ -0,0 +1,184 @@ +# Integration tests for repository-s3 +--- +"Snapshot/Restore with repository-s3 using temporary credentials": + + # Register repository with temporary credentials + - do: + snapshot.create_repository: + repository: repository_temporary + body: + type: s3 + settings: + bucket: ${temporary_bucket} + client: integration_test_temporary + base_path: ${temporary_base_path} + canned_acl: private + storage_class: standard + + - match: { acknowledged: true } + + # Get repository + - do: + snapshot.get_repository: + repository: repository_temporary + + - match: { repository_temporary.settings.bucket : ${temporary_bucket} } + - match: { repository_temporary.settings.client : "integration_test_temporary" } + - match: { repository_temporary.settings.base_path : ${temporary_base_path} } + - match: { repository_temporary.settings.canned_acl : "private" } + - match: { repository_temporary.settings.storage_class : "standard" } + - is_false: repository_temporary.settings.access_key + - is_false: repository_temporary.settings.secret_key + - is_false: repository_temporary.settings.session_token + + # Index documents + - do: + bulk: + refresh: true + body: + - index: + _index: docs + _type: doc + _id: 1 + - snapshot: one + - index: + _index: docs + _type: doc + _id: 2 + - snapshot: one + - index: + _index: docs + _type: doc + _id: 3 + - snapshot: one + + - do: + count: + index: docs + + - match: {count: 3} + + # Create a first snapshot + - do: + snapshot.create: + repository: repository_temporary + snapshot: snapshot-one + wait_for_completion: true + + - match: { snapshot.snapshot: snapshot-one } + - match: { snapshot.state : SUCCESS } + - match: { snapshot.include_global_state: true } + - match: { snapshot.shards.failed : 0 } + + - do: + snapshot.status: + repository: repository_temporary + snapshot: snapshot-one + + - is_true: snapshots + - match: { snapshots.0.snapshot: snapshot-one } + - match: { snapshots.0.state : SUCCESS } + + # Index more documents + - do: + bulk: + refresh: true + body: + - index: + _index: docs + _type: doc + _id: 4 + - snapshot: two + - index: + _index: docs + _type: doc + _id: 5 + - snapshot: two + - index: + _index: docs + _type: doc + _id: 6 + - snapshot: two + - index: + _index: docs + _type: doc + _id: 7 + - snapshot: two + + - do: + count: + index: docs + + - match: {count: 7} + + # Create a second snapshot + - do: + snapshot.create: + repository: repository_temporary + snapshot: snapshot-two + wait_for_completion: true + + - match: { snapshot.snapshot: snapshot-two } + - match: { snapshot.state : SUCCESS } + - match: { snapshot.shards.failed : 0 } + + - do: + snapshot.get: + repository: repository_temporary + snapshot: snapshot-one,snapshot-two + + - is_true: snapshots + - match: { snapshots.0.state : SUCCESS } + - match: { snapshots.1.state : SUCCESS } + + # Delete the index + - do: + indices.delete: + index: docs + + # Restore the second snapshot + - do: + snapshot.restore: + repository: repository_temporary + snapshot: snapshot-two + wait_for_completion: true + + - do: + count: + index: docs + + - match: {count: 7} + + # Delete the index again + - do: + indices.delete: + index: docs + + # Restore the first snapshot + - do: + snapshot.restore: + repository: repository_temporary + snapshot: snapshot-one + wait_for_completion: true + + - do: + count: + index: docs + + - match: {count: 3} + + # Remove the snapshots + - do: + snapshot.delete: + repository: repository_temporary + snapshot: snapshot-two + + - do: + snapshot.delete: + repository: repository_temporary + snapshot: snapshot-one + + # Remove our repository + - do: + snapshot.delete_repository: + repository: repository_temporary diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java index b372e9e0154f4..255cc6700159a 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java @@ -22,13 +22,11 @@ import com.amazonaws.ClientConfiguration; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.auth.InstanceProfileCredentialsProvider; import com.amazonaws.http.IdleConnectionReaper; import com.amazonaws.internal.StaticCredentialsProvider; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; - import org.apache.logging.log4j.Logger; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.MapBuilder; @@ -37,6 +35,7 @@ import java.io.IOException; import java.util.Map; + import static java.util.Collections.emptyMap; From 60e7b833c8a9fe03c135ee93202ad80d4f3e74a1 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 25 Jun 2018 17:55:29 +0100 Subject: [PATCH 16/21] Be pickier about auth verification in AmazonS3Fixture --- .../repositories/s3/AmazonS3Fixture.java | 34 ++++++++++++++++--- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java b/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java index 26aaf5b3661d9..0f54d4667d59c 100644 --- a/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java +++ b/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java @@ -68,12 +68,36 @@ protected Response handle(final Request request) throws IOException { final RequestHandler handler = handlers.retrieve(request.getMethod() + " " + request.getPath(), request.getParameters()); if (handler != null) { final String authorization = request.getHeader("Authorization"); - if (authorization == null - || (authorization.length() > 0 && authorization.contains("s3_integration_test_permanent_access_key") == false - && authorization.contains("s3_integration_test_temporary_access_key") == false)) { - return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Access Denied", ""); + final String permittedBucket; + if (authorization.contains("s3_integration_test_permanent_access_key")) { + final String sessionToken = request.getHeader("x-amz-security-token"); + if (sessionToken != null) { + return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Unexpected session token", ""); + } + permittedBucket = "permanent_bucket_test"; + } else if (authorization.contains("s3_integration_test_temporary_access_key")) { + final String sessionToken = request.getHeader("x-amz-security-token"); + if (sessionToken == null) { + return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "No session token", ""); + } + if (sessionToken.equals("s3_integration_test_temporary_session_token") == false) { + return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad session token", ""); + } + permittedBucket = "temporary_bucket_test"; + } else { + return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad access key", ""); + } + + if (handler != null) { + final String bucket = request.getParam("bucket"); + if (bucket != null && permittedBucket.equals(bucket) == false) { + // allow a null bucket to support bucket-free APIs like ListBuckets? + return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad bucket", ""); + } + return handler.handle(request); + } else { + return newInternalError(request.getId(), "No handler defined for request [" + request + "]"); } - return handler.handle(request); } return null; } From 63caae2bd2471326043affb1c9b026b9b9cd2ace Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 25 Jun 2018 18:04:12 +0100 Subject: [PATCH 17/21] Be more certain --- .../java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java b/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java index 0f54d4667d59c..4b06bcb3aa61f 100644 --- a/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java +++ b/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java @@ -91,7 +91,7 @@ protected Response handle(final Request request) throws IOException { if (handler != null) { final String bucket = request.getParam("bucket"); if (bucket != null && permittedBucket.equals(bucket) == false) { - // allow a null bucket to support bucket-free APIs like ListBuckets? + // allow a null bucket to support bucket-free APIs return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad bucket", ""); } return handler.handle(request); From d6abddc216fda220a5f91daef1809b6d998c113f Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 25 Jun 2018 18:40:04 +0100 Subject: [PATCH 18/21] Less hard-coding --- .../elasticsearch/repositories/s3/AmazonS3Fixture.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java b/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java index 4b06bcb3aa61f..4ca9372c11a9c 100644 --- a/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java +++ b/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java @@ -52,12 +52,17 @@ public class AmazonS3Fixture extends AbstractHttpFixture { /** Request handlers for the requests made by the S3 client **/ private final PathTrie handlers; + private final String permanentBucketName; + private final String temporaryBucketName; /** * Creates a {@link AmazonS3Fixture} */ private AmazonS3Fixture(final String workingDir, final String permanentBucketName, final String temporaryBucketName) { super(workingDir); + this.permanentBucketName = permanentBucketName; + this.temporaryBucketName = temporaryBucketName; + this.buckets.put(permanentBucketName, new Bucket(permanentBucketName)); this.buckets.put(temporaryBucketName, new Bucket(temporaryBucketName)); this.handlers = defaultHandlers(buckets); @@ -74,7 +79,7 @@ protected Response handle(final Request request) throws IOException { if (sessionToken != null) { return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Unexpected session token", ""); } - permittedBucket = "permanent_bucket_test"; + permittedBucket = permanentBucketName; } else if (authorization.contains("s3_integration_test_temporary_access_key")) { final String sessionToken = request.getHeader("x-amz-security-token"); if (sessionToken == null) { @@ -83,7 +88,7 @@ protected Response handle(final Request request) throws IOException { if (sessionToken.equals("s3_integration_test_temporary_session_token") == false) { return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad session token", ""); } - permittedBucket = "temporary_bucket_test"; + permittedBucket = temporaryBucketName; } else { return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad access key", ""); } From aafb0aaff42671f65e075abd0f3c233886641911 Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 26 Jun 2018 08:23:08 +0100 Subject: [PATCH 19/21] Tidy imports --- .../ec2/Ec2DiscoveryPluginTests.java | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java index 0db9e9dc8235b..720ffaddd74a5 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java @@ -19,6 +19,14 @@ package org.elasticsearch.discovery.ec2; +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.auth.BasicSessionCredentials; +import org.elasticsearch.common.settings.MockSecureSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.node.Node; +import org.elasticsearch.test.ESTestCase; + import java.io.IOException; import java.io.UncheckedIOException; import java.nio.file.Files; @@ -29,17 +37,6 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.auth.BasicSessionCredentials; -import org.elasticsearch.cluster.ClusterStateTaskConfig.Basic; -import org.elasticsearch.discovery.ec2.AwsEc2Service; -import org.elasticsearch.common.settings.MockSecureSettings; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.discovery.ec2.Ec2DiscoveryPlugin; -import org.elasticsearch.node.Node; -import org.elasticsearch.test.ESTestCase; - public class Ec2DiscoveryPluginTests extends ESTestCase { private Settings getNodeAttributes(Settings settings, String url) { From ff2969685d9cb3034a4f7bd5c4c23b886c542003 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 2 Jul 2018 13:49:09 +0100 Subject: [PATCH 20/21] Merge #31601 --- .../10_repository_permanent_credentials.yml | 63 ++++++++++++++++++- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository_permanent_credentials.yml b/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository_permanent_credentials.yml index d7566af9a848c..bb934d0931ca9 100644 --- a/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository_permanent_credentials.yml +++ b/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository_permanent_credentials.yml @@ -1,6 +1,7 @@ # Integration tests for repository-s3 + --- -"Snapshot/Restore with repository-s3 using permanent credentials": +setup: # Register repository with permanent credentials - do: @@ -15,7 +16,8 @@ canned_acl: private storage_class: standard - - match: { acknowledged: true } +--- +"Snapshot/Restore with repository-s3 using permanent credentials": # Get repository - do: @@ -178,6 +180,63 @@ repository: repository_permanent snapshot: snapshot-one +--- +"Register a repository with a non existing bucket": + + - do: + catch: /repository_exception/ + snapshot.create_repository: + repository: repository_permanent + body: + type: s3 + settings: + bucket: zHHkfSqlbnBsbpSgvCYtxrEfFLqghXtyPvvvKPNBnRCicNHQLE + client: integration_test + +--- +"Register a repository with a non existing client": + + - do: + catch: /repository_exception/ + snapshot.create_repository: + repository: repository_permanent + body: + type: s3 + settings: + bucket: repository_permanent + client: unknown + +--- +"Get a non existing snapshot": + + - do: + catch: /snapshot_missing_exception/ + snapshot.get: + repository: repository_permanent + snapshot: missing + +--- +"Delete a non existing snapshot": + + - do: + catch: /snapshot_missing_exception/ + snapshot.delete: + repository: repository_permanent + snapshot: missing + +--- +"Restore a non existing snapshot": + + - do: + catch: /snapshot_restore_exception/ + snapshot.restore: + repository: repository_permanent + snapshot: missing + wait_for_completion: true + +--- +teardown: + # Remove our repository - do: snapshot.delete_repository: From 86fa97a5669288b15cbe3051f429581c8a57fe44 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 2 Jul 2018 13:51:46 +0100 Subject: [PATCH 21/21] Merge #31601 for temp creds too --- .../20_repository_temporary_credentials.yml | 63 ++++++++++++++++++- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_temporary_credentials.yml b/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_temporary_credentials.yml index 1e3671eedb7d7..5da4f739cd522 100644 --- a/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_temporary_credentials.yml +++ b/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_temporary_credentials.yml @@ -1,6 +1,7 @@ # Integration tests for repository-s3 + --- -"Snapshot/Restore with repository-s3 using temporary credentials": +setup: # Register repository with temporary credentials - do: @@ -15,7 +16,8 @@ canned_acl: private storage_class: standard - - match: { acknowledged: true } +--- +"Snapshot/Restore with repository-s3 using temporary credentials": # Get repository - do: @@ -178,6 +180,63 @@ repository: repository_temporary snapshot: snapshot-one +--- +"Register a repository with a non existing bucket": + + - do: + catch: /repository_exception/ + snapshot.create_repository: + repository: repository_temporary + body: + type: s3 + settings: + bucket: zHHkfSqlbnBsbpSgvCYtxrEfFLqghXtyPvvvKPNBnRCicNHQLE + client: integration_test + +--- +"Register a repository with a non existing client": + + - do: + catch: /repository_exception/ + snapshot.create_repository: + repository: repository_temporary + body: + type: s3 + settings: + bucket: repository_temporary + client: unknown + +--- +"Get a non existing snapshot": + + - do: + catch: /snapshot_missing_exception/ + snapshot.get: + repository: repository_temporary + snapshot: missing + +--- +"Delete a non existing snapshot": + + - do: + catch: /snapshot_missing_exception/ + snapshot.delete: + repository: repository_temporary + snapshot: missing + +--- +"Restore a non existing snapshot": + + - do: + catch: /snapshot_restore_exception/ + snapshot.restore: + repository: repository_temporary + snapshot: missing + wait_for_completion: true + +--- +teardown: + # Remove our repository - do: snapshot.delete_repository: