From 9ae6905657b310ede5d968d71f4f3265eb26db5b Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Thu, 19 Jul 2018 09:17:49 -0700 Subject: [PATCH] add support for write index resolution when creating/updating documents (#31520) Now write operations like Index, Delete, Update rely on the write-index associated with an alias to operate against. This means writes will be accepted even when an alias points to multiple indices, so long as one is the write index. Routing values will be used from the AliasMetaData for the alias in the write-index. All read operations are left untouched. --- .../action/bulk/TransportBulkAction.java | 4 +- .../action/index/IndexRequest.java | 2 +- .../action/update/TransportUpdateAction.java | 2 +- .../metadata/IndexNameExpressionResolver.java | 95 ++++++++--- .../cluster/metadata/MetaData.java | 36 ++++ .../action/bulk/BulkIntegrationIT.java | 41 +++++ .../elasticsearch/aliases/IndexAliasesIT.java | 51 ++++++ .../IndexNameExpressionResolverTests.java | 154 ++++++++++++++++++ .../cluster/metadata/MetaDataTests.java | 81 +++++++++ .../org/elasticsearch/get/GetActionIT.java | 26 ++- .../org/elasticsearch/update/UpdateIT.java | 3 +- .../test/security/authz/12_index_alias.yml | 71 ++++++++ 12 files changed, 531 insertions(+), 35 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index a6ed8de653007..939b0b7024904 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -295,7 +295,7 @@ protected void doRun() throws Exception { TransportUpdateAction.resolveAndValidateRouting(metaData, concreteIndex.getName(), (UpdateRequest) docWriteRequest); break; case DELETE: - docWriteRequest.routing(metaData.resolveIndexRouting(docWriteRequest.routing(), docWriteRequest.index())); + docWriteRequest.routing(metaData.resolveWriteIndexRouting(docWriteRequest.routing(), docWriteRequest.index())); // check if routing is required, if so, throw error if routing wasn't specified if (docWriteRequest.routing() == null && metaData.routingRequired(concreteIndex.getName(), docWriteRequest.type())) { throw new RoutingMissingException(concreteIndex.getName(), docWriteRequest.type(), docWriteRequest.id()); @@ -474,7 +474,7 @@ Index getConcreteIndex(String indexOrAlias) { Index resolveIfAbsent(DocWriteRequest request) { Index concreteIndex = indices.get(request.index()); if (concreteIndex == null) { - concreteIndex = indexNameExpressionResolver.concreteSingleIndex(state, request); + concreteIndex = indexNameExpressionResolver.concreteWriteIndex(state, request); indices.put(request.index(), concreteIndex); } return concreteIndex; diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java index 51997b32edf1d..57e8ea6613817 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -496,7 +496,7 @@ public void process(Version indexCreatedVersion, @Nullable MappingMetaData mappi /* resolve the routing if needed */ public void resolveRouting(MetaData metaData) { - routing(metaData.resolveIndexRouting(routing, index)); + routing(metaData.resolveWriteIndexRouting(routing, index)); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java b/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java index 299a2ce812396..cc682619cbda5 100644 --- a/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java +++ b/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java @@ -104,7 +104,7 @@ protected void resolveRequest(ClusterState state, UpdateRequest request) { } public static void resolveAndValidateRouting(MetaData metaData, String concreteIndex, UpdateRequest request) { - request.routing((metaData.resolveIndexRouting(request.routing(), request.index()))); + request.routing((metaData.resolveWriteIndexRouting(request.routing(), request.index()))); // Fail fast on the node that received the request, rather than failing when translating on the index or delete request. if (request.routing() == null && metaData.routingRequired(concreteIndex, request.type())) { throw new RoutingMissingException(concreteIndex, request.type(), request.id()); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index 8fa3c2e0fc193..1f6a9fe027d1b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -42,7 +42,6 @@ import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -103,7 +102,7 @@ public String[] concreteIndexNames(ClusterState state, IndicesOptions options, S return concreteIndexNames(context, indexExpressions); } - /** + /** * Translates the provided index expression into actual concrete indices, properly deduplicated. * * @param state the cluster state containing all the data to resolve to expressions to concrete indices @@ -117,7 +116,7 @@ public String[] concreteIndexNames(ClusterState state, IndicesOptions options, S * indices options in the context don't allow such a case. */ public Index[] concreteIndices(ClusterState state, IndicesOptions options, String... indexExpressions) { - Context context = new Context(state, options); + Context context = new Context(state, options, false, false); return concreteIndices(context, indexExpressions); } @@ -193,30 +192,40 @@ Index[] concreteIndices(Context context, String... indexExpressions) { } } - Collection resolvedIndices = aliasOrIndex.getIndices(); - if (resolvedIndices.size() > 1 && !options.allowAliasesToMultipleIndices()) { - String[] indexNames = new String[resolvedIndices.size()]; - int i = 0; - for (IndexMetaData indexMetaData : resolvedIndices) { - indexNames[i++] = indexMetaData.getIndex().getName(); + if (aliasOrIndex.isAlias() && context.isResolveToWriteIndex()) { + AliasOrIndex.Alias alias = (AliasOrIndex.Alias) aliasOrIndex; + IndexMetaData writeIndex = alias.getWriteIndex(); + if (writeIndex == null) { + throw new IllegalArgumentException("no write index is defined for alias [" + alias.getAliasName() + "]." + + " The write index may be explicitly disabled using is_write_index=false or the alias points to multiple" + + " indices without one being designated as a write index"); } - throw new IllegalArgumentException("Alias [" + expression + "] has more than one indices associated with it [" + + concreteIndices.add(writeIndex.getIndex()); + } else { + if (aliasOrIndex.getIndices().size() > 1 && !options.allowAliasesToMultipleIndices()) { + String[] indexNames = new String[aliasOrIndex.getIndices().size()]; + int i = 0; + for (IndexMetaData indexMetaData : aliasOrIndex.getIndices()) { + indexNames[i++] = indexMetaData.getIndex().getName(); + } + throw new IllegalArgumentException("Alias [" + expression + "] has more than one indices associated with it [" + Arrays.toString(indexNames) + "], can't execute a single index op"); - } + } - for (IndexMetaData index : resolvedIndices) { - if (index.getState() == IndexMetaData.State.CLOSE) { - if (failClosed) { - throw new IndexClosedException(index.getIndex()); - } else { - if (options.forbidClosedIndices() == false) { - concreteIndices.add(index.getIndex()); + for (IndexMetaData index : aliasOrIndex.getIndices()) { + if (index.getState() == IndexMetaData.State.CLOSE) { + if (failClosed) { + throw new IndexClosedException(index.getIndex()); + } else { + if (options.forbidClosedIndices() == false) { + concreteIndices.add(index.getIndex()); + } } + } else if (index.getState() == IndexMetaData.State.OPEN) { + concreteIndices.add(index.getIndex()); + } else { + throw new IllegalStateException("index state [" + index.getState() + "] not supported"); } - } else if (index.getState() == IndexMetaData.State.OPEN) { - concreteIndices.add(index.getIndex()); - } else { - throw new IllegalStateException("index state [" + index.getState() + "] not supported"); } } } @@ -255,6 +264,28 @@ public Index concreteSingleIndex(ClusterState state, IndicesRequest request) { return indices[0]; } + /** + * Utility method that allows to resolve an index expression to its corresponding single write index. + * + * @param state the cluster state containing all the data to resolve to expression to a concrete index + * @param request The request that defines how the an alias or an index need to be resolved to a concrete index + * and the expression that can be resolved to an alias or an index name. + * @throws IllegalArgumentException if the index resolution does not lead to an index, or leads to more than one index + * @return the write index obtained as a result of the index resolution + */ + public Index concreteWriteIndex(ClusterState state, IndicesRequest request) { + if (request.indices() == null || (request.indices() != null && request.indices().length != 1)) { + throw new IllegalArgumentException("indices request must specify a single index expression"); + } + Context context = new Context(state, request.indicesOptions(), false, true); + Index[] indices = concreteIndices(context, request.indices()[0]); + if (indices.length != 1) { + throw new IllegalArgumentException("The index expression [" + request.indices()[0] + + "] and options provided did not point to a single write-index"); + } + return indices[0]; + } + /** * @return whether the specified alias or index exists. If the alias or index contains datemath then that is resolved too. */ @@ -292,7 +323,7 @@ public String[] indexAliases(ClusterState state, String index, Predicate resolvedExpressions = expressions != null ? Arrays.asList(expressions) : Collections.emptyList(); - Context context = new Context(state, IndicesOptions.lenientExpandOpen(), true); + Context context = new Context(state, IndicesOptions.lenientExpandOpen(), true, false); for (ExpressionResolver expressionResolver : expressionResolvers) { resolvedExpressions = expressionResolver.resolve(context, resolvedExpressions); } @@ -512,24 +543,26 @@ static final class Context { private final IndicesOptions options; private final long startTime; private final boolean preserveAliases; + private final boolean resolveToWriteIndex; Context(ClusterState state, IndicesOptions options) { this(state, options, System.currentTimeMillis()); } - Context(ClusterState state, IndicesOptions options, boolean preserveAliases) { - this(state, options, System.currentTimeMillis(), preserveAliases); + Context(ClusterState state, IndicesOptions options, boolean preserveAliases, boolean resolveToWriteIndex) { + this(state, options, System.currentTimeMillis(), preserveAliases, resolveToWriteIndex); } Context(ClusterState state, IndicesOptions options, long startTime) { - this(state, options, startTime, false); + this(state, options, startTime, false, false); } - Context(ClusterState state, IndicesOptions options, long startTime, boolean preserveAliases) { + Context(ClusterState state, IndicesOptions options, long startTime, boolean preserveAliases, boolean resolveToWriteIndex) { this.state = state; this.options = options; this.startTime = startTime; this.preserveAliases = preserveAliases; + this.resolveToWriteIndex = resolveToWriteIndex; } public ClusterState getState() { @@ -552,6 +585,14 @@ public long getStartTime() { boolean isPreserveAliases() { return preserveAliases; } + + /** + * This is used to require that aliases resolve to their write-index. It is currently not used in conjunction + * with preserveAliases. + */ + boolean isResolveToWriteIndex() { + return resolveToWriteIndex; + } } private interface ExpressionResolver { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index 4ed2adc9a1c9f..c024388868359 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -471,6 +471,42 @@ public String[] getConcreteAllClosedIndices() { return allClosedIndices; } + /** + * Returns indexing routing for the given aliasOrIndex. Resolves routing from the alias metadata used + * in the write index. + */ + public String resolveWriteIndexRouting(@Nullable String routing, String aliasOrIndex) { + if (aliasOrIndex == null) { + return routing; + } + + AliasOrIndex result = getAliasAndIndexLookup().get(aliasOrIndex); + if (result == null || result.isAlias() == false) { + return routing; + } + AliasOrIndex.Alias alias = (AliasOrIndex.Alias) result; + IndexMetaData writeIndex = alias.getWriteIndex(); + if (writeIndex == null) { + throw new IllegalArgumentException("alias [" + aliasOrIndex + "] does not have a write index"); + } + AliasMetaData aliasMd = writeIndex.getAliases().get(alias.getAliasName()); + if (aliasMd.indexRouting() != null) { + if (aliasMd.indexRouting().indexOf(',') != -1) { + throw new IllegalArgumentException("index/alias [" + aliasOrIndex + "] provided with routing value [" + + aliasMd.getIndexRouting() + "] that resolved to several routing values, rejecting operation"); + } + if (routing != null) { + if (!routing.equals(aliasMd.indexRouting())) { + throw new IllegalArgumentException("Alias [" + aliasOrIndex + "] has index routing associated with it [" + + aliasMd.indexRouting() + "], and was provided with routing value [" + routing + "], rejecting operation"); + } + } + // Alias routing overrides the parent routing (if any). + return aliasMd.indexRouting(); + } + return routing; + } + /** * Returns indexing routing for the given index. */ diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java index 8fcc76e018a6c..1fd912e72a426 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java @@ -20,13 +20,20 @@ package org.elasticsearch.action.bulk; +import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESIntegTestCase; import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.Map; import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; +import static org.hamcrest.Matchers.equalTo; public class BulkIntegrationIT extends ESIntegTestCase { public void testBulkIndexCreatesMapping() throws Exception { @@ -40,4 +47,38 @@ public void testBulkIndexCreatesMapping() throws Exception { assertTrue(mappingsResponse.getMappings().get("logstash-2014.03.30").containsKey("logs")); }); } + + /** + * This tests that the {@link TransportBulkAction} evaluates alias routing values correctly when dealing with + * an alias pointing to multiple indices, while a write index exits. + */ + public void testBulkWithWriteIndexAndRouting() { + Map twoShardsSettings = Collections.singletonMap(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2); + client().admin().indices().prepareCreate("index1") + .addAlias(new Alias("alias1").indexRouting("0")).setSettings(twoShardsSettings).get(); + client().admin().indices().prepareCreate("index2") + .addAlias(new Alias("alias1").indexRouting("0").writeIndex(randomFrom(false, null))) + .setSettings(twoShardsSettings).get(); + client().admin().indices().prepareCreate("index3") + .addAlias(new Alias("alias1").indexRouting("1").writeIndex(true)).setSettings(twoShardsSettings).get(); + + IndexRequest indexRequestWithAlias = new IndexRequest("alias1", "type", "id"); + if (randomBoolean()) { + indexRequestWithAlias.routing("1"); + } + indexRequestWithAlias.source(Collections.singletonMap("foo", "baz")); + BulkResponse bulkResponse = client().prepareBulk().add(indexRequestWithAlias).get(); + assertThat(bulkResponse.getItems()[0].getResponse().getIndex(), equalTo("index3")); + assertThat(bulkResponse.getItems()[0].getResponse().getShardId().getId(), equalTo(0)); + assertThat(bulkResponse.getItems()[0].getResponse().getVersion(), equalTo(1L)); + assertThat(bulkResponse.getItems()[0].getResponse().status(), equalTo(RestStatus.CREATED)); + assertThat(client().prepareGet("index3", "type", "id").setRouting("1").get().getSource().get("foo"), equalTo("baz")); + + bulkResponse = client().prepareBulk().add(client().prepareUpdate("alias1", "type", "id").setDoc("foo", "updated")).get(); + assertFalse(bulkResponse.hasFailures()); + assertThat(client().prepareGet("index3", "type", "id").setRouting("1").get().getSource().get("foo"), equalTo("updated")); + bulkResponse = client().prepareBulk().add(client().prepareDelete("alias1", "type", "id")).get(); + assertFalse(bulkResponse.hasFailures()); + assertFalse(client().prepareGet("index3", "type", "id").setRouting("1").get().isExists()); + } } diff --git a/server/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java b/server/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java index d72b4c5f1ec16..e8c152abdc216 100644 --- a/server/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java +++ b/server/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java @@ -25,6 +25,7 @@ import org.elasticsearch.action.admin.indices.alias.exists.AliasesExistResponse; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; +import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; @@ -57,6 +58,7 @@ import java.util.concurrent.TimeUnit; import static org.elasticsearch.client.Requests.createIndexRequest; +import static org.elasticsearch.client.Requests.deleteRequest; import static org.elasticsearch.client.Requests.indexRequest; import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_METADATA_BLOCK; import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_READ_ONLY_BLOCK; @@ -85,6 +87,17 @@ public void testAliases() throws Exception { ensureGreen(); + logger.info("--> aliasing index [test] with [alias1]"); + assertAcked(admin().indices().prepareAliases().addAlias("test", "alias1", false)); + + logger.info("--> indexing against [alias1], should fail now"); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, + () -> client().index(indexRequest("alias1").type("type1").id("1").source(source("2", "test"), + XContentType.JSON)).actionGet()); + assertThat(exception.getMessage(), equalTo("no write index is defined for alias [alias1]." + + " The write index may be explicitly disabled using is_write_index=false or the alias points to multiple" + + " indices without one being designated as a write index")); + logger.info("--> aliasing index [test] with [alias1]"); assertAcked(admin().indices().prepareAliases().addAlias("test", "alias1")); @@ -98,6 +111,44 @@ public void testAliases() throws Exception { ensureGreen(); + logger.info("--> add index [test_x] with [alias1]"); + assertAcked(admin().indices().prepareAliases().addAlias("test_x", "alias1")); + + logger.info("--> indexing against [alias1], should fail now"); + exception = expectThrows(IllegalArgumentException.class, + () -> client().index(indexRequest("alias1").type("type1").id("1").source(source("2", "test"), + XContentType.JSON)).actionGet()); + assertThat(exception.getMessage(), equalTo("no write index is defined for alias [alias1]." + + " The write index may be explicitly disabled using is_write_index=false or the alias points to multiple" + + " indices without one being designated as a write index")); + + logger.info("--> deleting against [alias1], should fail now"); + exception = expectThrows(IllegalArgumentException.class, + () -> client().delete(deleteRequest("alias1").type("type1").id("1")).actionGet()); + assertThat(exception.getMessage(), equalTo("no write index is defined for alias [alias1]." + + " The write index may be explicitly disabled using is_write_index=false or the alias points to multiple" + + " indices without one being designated as a write index")); + + logger.info("--> remove aliasing index [test_x] with [alias1]"); + assertAcked(admin().indices().prepareAliases().removeAlias("test_x", "alias1")); + + logger.info("--> indexing against [alias1], should work now"); + indexResponse = client().index(indexRequest("alias1").type("type1").id("1") + .source(source("1", "test"), XContentType.JSON)).actionGet(); + assertThat(indexResponse.getIndex(), equalTo("test")); + + logger.info("--> add index [test_x] with [alias1] as write-index"); + assertAcked(admin().indices().prepareAliases().addAlias("test_x", "alias1", true)); + + logger.info("--> indexing against [alias1], should work now"); + indexResponse = client().index(indexRequest("alias1").type("type1").id("1") + .source(source("1", "test"), XContentType.JSON)).actionGet(); + assertThat(indexResponse.getIndex(), equalTo("test_x")); + + logger.info("--> deleting against [alias1], should fail now"); + DeleteResponse deleteResponse = client().delete(deleteRequest("alias1").type("type1").id("1")).actionGet(); + assertThat(deleteResponse.getIndex(), equalTo("test_x")); + logger.info("--> remove [alias1], Aliasing index [test_x] with [alias1]"); assertAcked(admin().indices().prepareAliases().removeAlias("test", "alias1").addAlias("test_x", "alias1")); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java index 0530bd617af63..9ad9603b1489b 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java @@ -20,14 +20,20 @@ package org.elasticsearch.cluster.metadata; import org.elasticsearch.Version; +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData.State; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.indices.IndexClosedException; import org.elasticsearch.indices.InvalidIndexNameException; @@ -37,6 +43,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.function.Function; import static org.elasticsearch.common.util.set.Sets.newHashSet; import static org.hamcrest.Matchers.arrayContaining; @@ -44,6 +51,7 @@ import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.emptyArray; +import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; @@ -996,6 +1004,152 @@ public void testIndexAliases() { assertArrayEquals(new String[] {"test-alias-0", "test-alias-1", "test-alias-non-filtering"}, strings); } + public void testConcreteWriteIndexSuccessful() { + boolean testZeroWriteIndex = randomBoolean(); + MetaData.Builder mdBuilder = MetaData.builder() + .put(indexBuilder("test-0").state(State.OPEN) + .putAlias(AliasMetaData.builder("test-alias").writeIndex(testZeroWriteIndex ? true : null))); + ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build(); + String[] strings = indexNameExpressionResolver + .indexAliases(state, "test-0", x -> true, true, "test-*"); + Arrays.sort(strings); + assertArrayEquals(new String[] {"test-alias"}, strings); + IndicesRequest request = new IndicesRequest() { + + @Override + public String[] indices() { + return new String[] { "test-alias" }; + } + + @Override + public IndicesOptions indicesOptions() { + return IndicesOptions.strictSingleIndexNoExpandForbidClosed(); + } + }; + Index writeIndex = indexNameExpressionResolver.concreteWriteIndex(state, request); + assertThat(writeIndex.getName(), equalTo("test-0")); + + state = ClusterState.builder(state).metaData(MetaData.builder(state.metaData()) + .put(indexBuilder("test-1").putAlias(AliasMetaData.builder("test-alias") + .writeIndex(testZeroWriteIndex ? randomFrom(false, null) : true)))).build(); + writeIndex = indexNameExpressionResolver.concreteWriteIndex(state, request); + assertThat(writeIndex.getName(), equalTo(testZeroWriteIndex ? "test-0" : "test-1")); + } + + public void testConcreteWriteIndexWithInvalidIndicesRequest() { + MetaData.Builder mdBuilder = MetaData.builder() + .put(indexBuilder("test-0").state(State.OPEN) + .putAlias(AliasMetaData.builder("test-alias"))); + ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build(); + Function requestGen = (indices) -> new IndicesRequest() { + + @Override + public String[] indices() { + return indices; + } + + @Override + public IndicesOptions indicesOptions() { + return IndicesOptions.strictSingleIndexNoExpandForbidClosed(); + } + }; + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, + () -> indexNameExpressionResolver.concreteWriteIndex(state, requestGen.apply(null))); + assertThat(exception.getMessage(), equalTo("indices request must specify a single index expression")); + exception = expectThrows(IllegalArgumentException.class, + () -> indexNameExpressionResolver.concreteWriteIndex(state, requestGen.apply(new String[] {"too", "many"}))); + assertThat(exception.getMessage(), equalTo("indices request must specify a single index expression")); + + + } + + public void testConcreteWriteIndexWithWildcardExpansion() { + boolean testZeroWriteIndex = randomBoolean(); + MetaData.Builder mdBuilder = MetaData.builder() + .put(indexBuilder("test-1").state(State.OPEN) + .putAlias(AliasMetaData.builder("test-alias").writeIndex(testZeroWriteIndex ? true : null))) + .put(indexBuilder("test-0").state(State.OPEN) + .putAlias(AliasMetaData.builder("test-alias").writeIndex(testZeroWriteIndex ? randomFrom(false, null) : true))); + ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build(); + String[] strings = indexNameExpressionResolver + .indexAliases(state, "test-0", x -> true, true, "test-*"); + Arrays.sort(strings); + assertArrayEquals(new String[] {"test-alias"}, strings); + IndicesRequest request = new IndicesRequest() { + + @Override + public String[] indices() { + return new String[] { "test-*"}; + } + + @Override + public IndicesOptions indicesOptions() { + return IndicesOptions.strictExpandOpenAndForbidClosed(); + } + }; + + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, + () -> indexNameExpressionResolver.concreteWriteIndex(state, request)); + assertThat(exception.getMessage(), + equalTo("The index expression [test-*] and options provided did not point to a single write-index")); + } + + public void testConcreteWriteIndexWithNoWriteIndexWithSingleIndex() { + MetaData.Builder mdBuilder = MetaData.builder() + .put(indexBuilder("test-0").state(State.OPEN) + .putAlias(AliasMetaData.builder("test-alias").writeIndex(false))); + ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build(); + String[] strings = indexNameExpressionResolver + .indexAliases(state, "test-0", x -> true, true, "test-*"); + Arrays.sort(strings); + assertArrayEquals(new String[] {"test-alias"}, strings); + DocWriteRequest request = randomFrom(new IndexRequest("test-alias"), + new UpdateRequest("test-alias", "_type", "_id"), new DeleteRequest("test-alias")); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, + () -> indexNameExpressionResolver.concreteWriteIndex(state, request)); + assertThat(exception.getMessage(), equalTo("no write index is defined for alias [test-alias]." + + " The write index may be explicitly disabled using is_write_index=false or the alias points to multiple" + + " indices without one being designated as a write index")); + } + + public void testConcreteWriteIndexWithNoWriteIndexWithMultipleIndices() { + MetaData.Builder mdBuilder = MetaData.builder() + .put(indexBuilder("test-0").state(State.OPEN) + .putAlias(AliasMetaData.builder("test-alias").writeIndex(randomFrom(false, null)))) + .put(indexBuilder("test-1").state(State.OPEN) + .putAlias(AliasMetaData.builder("test-alias").writeIndex(randomFrom(false, null)))); + ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build(); + String[] strings = indexNameExpressionResolver + .indexAliases(state, "test-0", x -> true, true, "test-*"); + Arrays.sort(strings); + assertArrayEquals(new String[] {"test-alias"}, strings); + DocWriteRequest request = randomFrom(new IndexRequest("test-alias"), + new UpdateRequest("test-alias", "_type", "_id"), new DeleteRequest("test-alias")); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, + () -> indexNameExpressionResolver.concreteWriteIndex(state, request)); + assertThat(exception.getMessage(), equalTo("no write index is defined for alias [test-alias]." + + " The write index may be explicitly disabled using is_write_index=false or the alias points to multiple" + + " indices without one being designated as a write index")); + } + + public void testAliasResolutionNotAllowingMultipleIndices() { + boolean test0WriteIndex = randomBoolean(); + MetaData.Builder mdBuilder = MetaData.builder() + .put(indexBuilder("test-0").state(State.OPEN) + .putAlias(AliasMetaData.builder("test-alias").writeIndex(randomFrom(test0WriteIndex, null)))) + .put(indexBuilder("test-1").state(State.OPEN) + .putAlias(AliasMetaData.builder("test-alias").writeIndex(randomFrom(!test0WriteIndex, null)))); + ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build(); + String[] strings = indexNameExpressionResolver + .indexAliases(state, "test-0", x -> true, true, "test-*"); + Arrays.sort(strings); + assertArrayEquals(new String[] {"test-alias"}, strings); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, + () -> indexNameExpressionResolver.concreteIndexNames(state, IndicesOptions.strictSingleIndexNoExpandForbidClosed(), + "test-alias")); + assertThat(exception.getMessage(), endsWith(", can't execute a single index op")); + } + public void testDeleteIndexIgnoresAliases() { MetaData.Builder mdBuilder = MetaData.builder() .put(indexBuilder("test-index").state(State.OPEN) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java index 32dd4324ff835..38e3fcc6ea7c5 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java @@ -172,6 +172,87 @@ public void testResolveIndexRouting() { } catch (IllegalArgumentException ex) { assertThat(ex.getMessage(), is("index/alias [alias2] provided with routing value [1,2] that resolved to several routing values, rejecting operation")); } + + IndexMetaData.Builder builder2 = IndexMetaData.builder("index2") + .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0) + .putAlias(AliasMetaData.builder("alias0").build()); + MetaData metaDataTwoIndices = MetaData.builder(metaData).put(builder2).build(); + + // alias with multiple indices + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, + () -> metaDataTwoIndices.resolveIndexRouting("1", "alias0")); + assertThat(exception.getMessage(), startsWith("Alias [alias0] has more than one index associated with it")); + } + + public void testResolveWriteIndexRouting() { + AliasMetaData.Builder aliasZeroBuilder = AliasMetaData.builder("alias0"); + if (randomBoolean()) { + aliasZeroBuilder.writeIndex(true); + } + IndexMetaData.Builder builder = IndexMetaData.builder("index") + .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0) + .putAlias(aliasZeroBuilder.build()) + .putAlias(AliasMetaData.builder("alias1").routing("1").build()) + .putAlias(AliasMetaData.builder("alias2").routing("1,2").build()) + .putAlias(AliasMetaData.builder("alias3").writeIndex(false).build()) + .putAlias(AliasMetaData.builder("alias4").routing("1,2").writeIndex(true).build()); + MetaData metaData = MetaData.builder().put(builder).build(); + + // no alias, no index + assertEquals(metaData.resolveWriteIndexRouting(null, null), null); + assertEquals(metaData.resolveWriteIndexRouting("0", null), "0"); + + // index, no alias + assertEquals(metaData.resolveWriteIndexRouting(null, "index"), null); + assertEquals(metaData.resolveWriteIndexRouting("0", "index"), "0"); + + // alias with no index routing + assertEquals(metaData.resolveWriteIndexRouting(null, "alias0"), null); + assertEquals(metaData.resolveWriteIndexRouting("0", "alias0"), "0"); + + // alias with index routing. + assertEquals(metaData.resolveWriteIndexRouting(null, "alias1"), "1"); + Exception exception = expectThrows(IllegalArgumentException.class, () -> metaData.resolveWriteIndexRouting("0", "alias1")); + assertThat(exception.getMessage(), + is("Alias [alias1] has index routing associated with it [1], and was provided with routing value [0], rejecting operation")); + + // alias with invalid index routing. + exception = expectThrows(IllegalArgumentException.class, () -> metaData.resolveWriteIndexRouting(null, "alias2")); + assertThat(exception.getMessage(), + is("index/alias [alias2] provided with routing value [1,2] that resolved to several routing values, rejecting operation")); + exception = expectThrows(IllegalArgumentException.class, () -> metaData.resolveWriteIndexRouting("1", "alias2")); + assertThat(exception.getMessage(), + is("index/alias [alias2] provided with routing value [1,2] that resolved to several routing values, rejecting operation")); + exception = expectThrows(IllegalArgumentException.class, () -> metaData.resolveWriteIndexRouting(randomFrom("1", null), "alias4")); + assertThat(exception.getMessage(), + is("index/alias [alias4] provided with routing value [1,2] that resolved to several routing values, rejecting operation")); + + // alias with no write index + exception = expectThrows(IllegalArgumentException.class, () -> metaData.resolveWriteIndexRouting("1", "alias3")); + assertThat(exception.getMessage(), + is("alias [alias3] does not have a write index")); + + + // aliases with multiple indices + AliasMetaData.Builder aliasZeroBuilderTwo = AliasMetaData.builder("alias0"); + if (randomBoolean()) { + aliasZeroBuilder.writeIndex(false); + } + IndexMetaData.Builder builder2 = IndexMetaData.builder("index2") + .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0) + .putAlias(aliasZeroBuilderTwo.build()) + .putAlias(AliasMetaData.builder("alias1").routing("0").writeIndex(true).build()) + .putAlias(AliasMetaData.builder("alias2").writeIndex(true).build()); + MetaData metaDataTwoIndices = MetaData.builder(metaData).put(builder2).build(); + + // verify that new write index is used + assertThat("0", equalTo(metaDataTwoIndices.resolveWriteIndexRouting("0", "alias1"))); } public void testUnknownFieldClusterMetaData() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/get/GetActionIT.java b/server/src/test/java/org/elasticsearch/get/GetActionIT.java index 30f86241cbd6d..5ed6b957c78a4 100644 --- a/server/src/test/java/org/elasticsearch/get/GetActionIT.java +++ b/server/src/test/java/org/elasticsearch/get/GetActionIT.java @@ -29,6 +29,7 @@ import org.elasticsearch.action.get.MultiGetRequest; import org.elasticsearch.action.get.MultiGetRequestBuilder; import org.elasticsearch.action.get.MultiGetResponse; +import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.support.DefaultShardOperationFailedException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; @@ -39,6 +40,7 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalSettingsPlugin; @@ -51,6 +53,7 @@ import static java.util.Collections.singleton; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.instanceOf; @@ -70,7 +73,7 @@ public void testSimpleGet() { assertAcked(prepareCreate("test") .addMapping("type1", "field1", "type=keyword,store=true", "field2", "type=keyword,store=true") .setSettings(Settings.builder().put("index.refresh_interval", -1)) - .addAlias(new Alias("alias"))); + .addAlias(new Alias("alias").writeIndex(randomFrom(true, false, null)))); ensureGreen(); GetResponse response = client().prepareGet(indexOrAlias(), "type1", "1").get(); @@ -192,12 +195,31 @@ public void testSimpleGet() { assertThat(response.isExists(), equalTo(false)); } + public void testGetWithAliasPointingToMultipleIndices() { + client().admin().indices().prepareCreate("index1") + .addAlias(new Alias("alias1").indexRouting("0")).get(); + if (randomBoolean()) { + client().admin().indices().prepareCreate("index2") + .addAlias(new Alias("alias1").indexRouting("0").writeIndex(randomFrom(false, null))).get(); + } else { + client().admin().indices().prepareCreate("index3") + .addAlias(new Alias("alias1").indexRouting("1").writeIndex(true)).get(); + } + IndexResponse indexResponse = client().prepareIndex("index1", "type", "id") + .setSource(Collections.singletonMap("foo", "bar")).get(); + assertThat(indexResponse.status().getStatus(), equalTo(RestStatus.CREATED.getStatus())); + + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> + client().prepareGet("alias1", "type", "_alias_id").get()); + assertThat(exception.getMessage(), endsWith("can't execute a single index op")); + } + private static String indexOrAlias() { return randomBoolean() ? "test" : "alias"; } public void testSimpleMultiGet() throws Exception { - assertAcked(prepareCreate("test").addAlias(new Alias("alias")) + assertAcked(prepareCreate("test").addAlias(new Alias("alias").writeIndex(randomFrom(true, false, null))) .addMapping("type1", "field", "type=keyword,store=true") .setSettings(Settings.builder().put("index.refresh_interval", -1))); ensureGreen(); diff --git a/server/src/test/java/org/elasticsearch/update/UpdateIT.java b/server/src/test/java/org/elasticsearch/update/UpdateIT.java index c86dfcb98f701..e4ea078b8f716 100644 --- a/server/src/test/java/org/elasticsearch/update/UpdateIT.java +++ b/server/src/test/java/org/elasticsearch/update/UpdateIT.java @@ -140,8 +140,7 @@ protected Collection> nodePlugins() { private void createTestIndex() throws Exception { logger.info("--> creating index test"); - - assertAcked(prepareCreate("test").addAlias(new Alias("alias"))); + assertAcked(prepareCreate("test").addAlias(new Alias("alias").writeIndex(randomFrom(true, null)))); } public void testUpsert() throws Exception { diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/12_index_alias.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/12_index_alias.yml index 44d91d691e1c2..1e947c5639d77 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/12_index_alias.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/12_index_alias.yml @@ -310,3 +310,74 @@ teardown: index: write_index_2 body: { "query": { "terms": { "_id": [ "19" ] } } } - match: { hits.total: 1 } + +--- +"Test bulk indexing into an alias when resolved to write index": + - do: + indices.update_aliases: + body: + actions: + - add: + index: write_index_2 + alias: can_write_2 + is_write_index: true + - add: + index: write_index_2 + alias: can_read_2 + is_write_index: true + - add: + index: write_index_1 + alias: can_write_3 + is_write_index: true + - add: + index: write_index_2 + alias: can_write_3 + is_write_index: false + + - do: + headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user + bulk: + refresh: true + body: + - '{"index": {"_index": "can_read_1", "_type": "doc", "_id": "20"}}' + - '{"name": "doc20"}' + - '{"index": {"_index": "can_write_1", "_type": "doc", "_id": "21"}}' + - '{"name": "doc21"}' + - '{"index": {"_index": "can_read_2", "_type": "doc", "_id": "22"}}' + - '{"name": "doc22"}' + - '{"index": {"_index": "can_write_2", "_type": "doc", "_id": "23"}}' + - '{"name": "doc23"}' + - '{"index": {"_index": "can_write_3", "_type": "doc", "_id": "24"}}' + - '{"name": "doc24"}' + - '{"update": {"_index": "can_write_3", "_type": "doc", "_id": "24"}}' + - '{"doc": { "name": "doc_24"}}' + - '{"delete": {"_index": "can_write_3", "_type": "doc", "_id": "24"}}' + - match: { errors: true } + - match: { items.0.index.status: 403 } + - match: { items.0.index.error.type: "security_exception" } + - match: { items.1.index.status: 201 } + - match: { items.2.index.status: 403 } + - match: { items.2.index.error.type: "security_exception" } + - match: { items.3.index.status: 403 } + - match: { items.3.index.error.type: "security_exception" } + - match: { items.4.index.status: 201 } + - match: { items.5.update.status: 200 } + - match: { items.6.delete.status: 200 } + + - do: # superuser + search: + index: write_index_1 + body: { "query": { "terms": { "_id": [ "21" ] } } } + - match: { hits.total: 1 } + + - do: + indices.delete_alias: + index: "write_index_2" + name: [ "can_write_2", "can_read_2" ] + ignore: 404 + + - do: + indices.delete_alias: + index: "write_index_1" + name: [ "can_write_3" ] + ignore: 404