diff --git a/core/src/main/java/org/elasticsearch/action/ActionModule.java b/core/src/main/java/org/elasticsearch/action/ActionModule.java index 86582e9b8d046..28fd3458b902a 100644 --- a/core/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/core/src/main/java/org/elasticsearch/action/ActionModule.java @@ -128,7 +128,9 @@ import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction; import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresAction; import org.elasticsearch.action.admin.indices.shards.TransportIndicesShardStoresAction; +import org.elasticsearch.action.admin.indices.shrink.ResizeAction; import org.elasticsearch.action.admin.indices.shrink.ShrinkAction; +import org.elasticsearch.action.admin.indices.shrink.TransportResizeAction; import org.elasticsearch.action.admin.indices.shrink.TransportShrinkAction; import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction; import org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction; @@ -181,7 +183,6 @@ import org.elasticsearch.action.search.TransportMultiSearchAction; import org.elasticsearch.action.search.TransportSearchAction; import org.elasticsearch.action.search.TransportSearchScrollAction; -import org.elasticsearch.action.support.ActionFilter; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.AutoCreateIndex; import org.elasticsearch.action.support.DestructiveOperations; @@ -199,7 +200,6 @@ import org.elasticsearch.common.NamedRegistry; import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.inject.multibindings.MapBinder; -import org.elasticsearch.common.inject.multibindings.Multibinder; import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; @@ -271,6 +271,7 @@ import org.elasticsearch.rest.action.admin.indices.RestRefreshAction; import org.elasticsearch.rest.action.admin.indices.RestRolloverIndexAction; import org.elasticsearch.rest.action.admin.indices.RestShrinkIndexAction; +import org.elasticsearch.rest.action.admin.indices.RestSplitIndexAction; import org.elasticsearch.rest.action.admin.indices.RestSyncedFlushAction; import org.elasticsearch.rest.action.admin.indices.RestUpdateSettingsAction; import org.elasticsearch.rest.action.admin.indices.RestUpgradeAction; @@ -324,7 +325,6 @@ import java.util.function.UnaryOperator; import java.util.stream.Collectors; -import static java.util.Collections.unmodifiableList; import static java.util.Collections.unmodifiableMap; /** @@ -438,6 +438,7 @@ public void reg actions.register(IndicesShardStoresAction.INSTANCE, TransportIndicesShardStoresAction.class); actions.register(CreateIndexAction.INSTANCE, TransportCreateIndexAction.class); actions.register(ShrinkAction.INSTANCE, TransportShrinkAction.class); + actions.register(ResizeAction.INSTANCE, TransportResizeAction.class); actions.register(RolloverAction.INSTANCE, TransportRolloverAction.class); actions.register(DeleteIndexAction.INSTANCE, TransportDeleteIndexAction.class); actions.register(GetIndexAction.INSTANCE, TransportGetIndexAction.class); @@ -554,6 +555,7 @@ public void initRestHandlers(Supplier nodesInCluster) { registerHandler.accept(new RestIndicesAliasesAction(settings, restController)); registerHandler.accept(new RestCreateIndexAction(settings, restController)); registerHandler.accept(new RestShrinkIndexAction(settings, restController)); + registerHandler.accept(new RestSplitIndexAction(settings, restController)); registerHandler.accept(new RestRolloverIndexAction(settings, restController)); registerHandler.accept(new RestDeleteIndexAction(settings, restController)); registerHandler.accept(new RestCloseIndexAction(settings, restController)); diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java index a2290a5e2556e..1734c340bd4ef 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java @@ -20,6 +20,7 @@ package org.elasticsearch.action.admin.indices.create; import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.cluster.ack.ClusterStateUpdateRequest; import org.elasticsearch.cluster.block.ClusterBlock; @@ -43,7 +44,8 @@ public class CreateIndexClusterStateUpdateRequest extends ClusterStateUpdateRequ private final String index; private final String providedName; private final boolean updateAllTypes; - private Index shrinkFrom; + private Index recoverFrom; + private ResizeType resizeType; private IndexMetaData.State state = IndexMetaData.State.OPEN; @@ -59,7 +61,6 @@ public class CreateIndexClusterStateUpdateRequest extends ClusterStateUpdateRequ private ActiveShardCount waitForActiveShards = ActiveShardCount.DEFAULT; - public CreateIndexClusterStateUpdateRequest(TransportMessage originalMessage, String cause, String index, String providedName, boolean updateAllTypes) { this.originalMessage = originalMessage; @@ -99,8 +100,8 @@ public CreateIndexClusterStateUpdateRequest state(IndexMetaData.State state) { return this; } - public CreateIndexClusterStateUpdateRequest shrinkFrom(Index shrinkFrom) { - this.shrinkFrom = shrinkFrom; + public CreateIndexClusterStateUpdateRequest recoverFrom(Index recoverFrom) { + this.recoverFrom = recoverFrom; return this; } @@ -109,6 +110,11 @@ public CreateIndexClusterStateUpdateRequest waitForActiveShards(ActiveShardCount return this; } + public CreateIndexClusterStateUpdateRequest resizeType(ResizeType resizeType) { + this.resizeType = resizeType; + return this; + } + public TransportMessage originalMessage() { return originalMessage; } @@ -145,8 +151,8 @@ public Set blocks() { return blocks; } - public Index shrinkFrom() { - return shrinkFrom; + public Index recoverFrom() { + return recoverFrom; } /** True if all fields that span multiple types should be updated, false otherwise */ @@ -165,4 +171,11 @@ public String getProvidedName() { public ActiveShardCount waitForActiveShards() { return waitForActiveShards; } + + /** + * Returns the resize type or null if this is an ordinary create index request + */ + public ResizeType resizeType() { + return resizeType; + } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeAction.java new file mode 100644 index 0000000000000..9447e0803e2ba --- /dev/null +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeAction.java @@ -0,0 +1,45 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.shrink; + +import org.elasticsearch.Version; +import org.elasticsearch.action.Action; +import org.elasticsearch.client.ElasticsearchClient; + +public class ResizeAction extends Action { + + public static final ResizeAction INSTANCE = new ResizeAction(); + public static final String NAME = "indices:admin/resize"; + public static final Version COMPATIBILITY_VERSION = Version.V_7_0_0_alpha1; // TODO remove this once it's backported + + private ResizeAction() { + super(NAME); + } + + @Override + public ResizeResponse newResponse() { + return new ResizeResponse(); + } + + @Override + public ResizeRequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new ResizeRequestBuilder(client, this); + } +} diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequest.java similarity index 65% rename from core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequest.java rename to core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequest.java index 6ea58200a4500..f2f648f70ffa9 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequest.java @@ -18,12 +18,14 @@ */ package org.elasticsearch.action.admin.indices.shrink; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -37,37 +39,41 @@ /** * Request class to shrink an index into a single shard */ -public class ShrinkRequest extends AcknowledgedRequest implements IndicesRequest { +public class ResizeRequest extends AcknowledgedRequest implements IndicesRequest { - public static final ObjectParser PARSER = new ObjectParser<>("shrink_request", null); + public static final ObjectParser PARSER = new ObjectParser<>("resize_request", null); static { - PARSER.declareField((parser, request, context) -> request.getShrinkIndexRequest().settings(parser.map()), + PARSER.declareField((parser, request, context) -> request.getTargetIndexRequest().settings(parser.map()), new ParseField("settings"), ObjectParser.ValueType.OBJECT); - PARSER.declareField((parser, request, context) -> request.getShrinkIndexRequest().aliases(parser.map()), + PARSER.declareField((parser, request, context) -> request.getTargetIndexRequest().aliases(parser.map()), new ParseField("aliases"), ObjectParser.ValueType.OBJECT); } - private CreateIndexRequest shrinkIndexRequest; + private CreateIndexRequest targetIndexRequest; private String sourceIndex; + private ResizeType type = ResizeType.SHRINK; - ShrinkRequest() {} + ResizeRequest() {} - public ShrinkRequest(String targetIndex, String sourceindex) { - this.shrinkIndexRequest = new CreateIndexRequest(targetIndex); - this.sourceIndex = sourceindex; + public ResizeRequest(String targetIndex, String sourceIndex) { + this.targetIndexRequest = new CreateIndexRequest(targetIndex); + this.sourceIndex = sourceIndex; } @Override public ActionRequestValidationException validate() { - ActionRequestValidationException validationException = shrinkIndexRequest == null ? null : shrinkIndexRequest.validate(); + ActionRequestValidationException validationException = targetIndexRequest == null ? null : targetIndexRequest.validate(); if (sourceIndex == null) { validationException = addValidationError("source index is missing", validationException); } - if (shrinkIndexRequest == null) { - validationException = addValidationError("shrink index request is missing", validationException); + if (targetIndexRequest == null) { + validationException = addValidationError("target index request is missing", validationException); } - if (shrinkIndexRequest.settings().getByPrefix("index.sort.").isEmpty() == false) { - validationException = addValidationError("can't override index sort when shrinking index", validationException); + if (targetIndexRequest.settings().getByPrefix("index.sort.").isEmpty() == false) { + validationException = addValidationError("can't override index sort when resizing an index", validationException); + } + if (type == ResizeType.SPLIT && IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.exists(targetIndexRequest.settings()) == false) { + validationException = addValidationError("index.number_of_shards is required for split operations", validationException); } return validationException; } @@ -79,16 +85,24 @@ public void setSourceIndex(String index) { @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - shrinkIndexRequest = new CreateIndexRequest(); - shrinkIndexRequest.readFrom(in); + targetIndexRequest = new CreateIndexRequest(); + targetIndexRequest.readFrom(in); sourceIndex = in.readString(); + if (in.getVersion().onOrAfter(ResizeAction.COMPATIBILITY_VERSION)) { + type = in.readEnum(ResizeType.class); + } else { + type = ResizeType.SHRINK; // BWC this used to be shrink only + } } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - shrinkIndexRequest.writeTo(out); + targetIndexRequest.writeTo(out); out.writeString(sourceIndex); + if (out.getVersion().onOrAfter(ResizeAction.COMPATIBILITY_VERSION)) { + out.writeEnum(type); + } } @Override @@ -101,15 +115,15 @@ public IndicesOptions indicesOptions() { return IndicesOptions.lenientExpandOpen(); } - public void setShrinkIndex(CreateIndexRequest shrinkIndexRequest) { - this.shrinkIndexRequest = Objects.requireNonNull(shrinkIndexRequest, "shrink index request must not be null"); + public void setTargetIndex(CreateIndexRequest targetIndexRequest) { + this.targetIndexRequest = Objects.requireNonNull(targetIndexRequest, "target index request must not be null"); } /** * Returns the {@link CreateIndexRequest} for the shrink index */ - public CreateIndexRequest getShrinkIndexRequest() { - return shrinkIndexRequest; + public CreateIndexRequest getTargetIndexRequest() { + return targetIndexRequest; } /** @@ -128,13 +142,13 @@ public String getSourceIndex() { * non-negative integer, up to the number of copies per shard (number of replicas + 1), * to wait for the desired amount of shard copies to become active before returning. * Index creation will only wait up until the timeout value for the number of shard copies - * to be active before returning. Check {@link ShrinkResponse#isShardsAcked()} to + * to be active before returning. Check {@link ResizeResponse#isShardsAcked()} to * determine if the requisite shard copies were all started before returning or timing out. * * @param waitForActiveShards number of active shard copies to wait on */ public void setWaitForActiveShards(ActiveShardCount waitForActiveShards) { - this.getShrinkIndexRequest().waitForActiveShards(waitForActiveShards); + this.getTargetIndexRequest().waitForActiveShards(waitForActiveShards); } /** @@ -145,4 +159,18 @@ public void setWaitForActiveShards(ActiveShardCount waitForActiveShards) { public void setWaitForActiveShards(final int waitForActiveShards) { setWaitForActiveShards(ActiveShardCount.from(waitForActiveShards)); } + + /** + * The type of the resize operation + */ + public void setResizeType(ResizeType type) { + this.type = Objects.requireNonNull(type); + } + + /** + * Returns the type of the resize operation + */ + public ResizeType getResizeType() { + return type; + } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestBuilder.java similarity index 73% rename from core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequestBuilder.java rename to core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestBuilder.java index 2bd10397193d5..6d8d98c0d75f0 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestBuilder.java @@ -18,31 +18,32 @@ */ package org.elasticsearch.action.admin.indices.shrink; +import org.elasticsearch.action.Action; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.settings.Settings; -public class ShrinkRequestBuilder extends AcknowledgedRequestBuilder { - public ShrinkRequestBuilder(ElasticsearchClient client, ShrinkAction action) { - super(client, action, new ShrinkRequest()); +public class ResizeRequestBuilder extends AcknowledgedRequestBuilder { + public ResizeRequestBuilder(ElasticsearchClient client, Action action) { + super(client, action, new ResizeRequest()); } - public ShrinkRequestBuilder setTargetIndex(CreateIndexRequest request) { - this.request.setShrinkIndex(request); + public ResizeRequestBuilder setTargetIndex(CreateIndexRequest request) { + this.request.setTargetIndex(request); return this; } - public ShrinkRequestBuilder setSourceIndex(String index) { + public ResizeRequestBuilder setSourceIndex(String index) { this.request.setSourceIndex(index); return this; } - public ShrinkRequestBuilder setSettings(Settings settings) { - this.request.getShrinkIndexRequest().settings(settings); + public ResizeRequestBuilder setSettings(Settings settings) { + this.request.getTargetIndexRequest().settings(settings); return this; } @@ -55,12 +56,12 @@ public ShrinkRequestBuilder setSettings(Settings settings) { * non-negative integer, up to the number of copies per shard (number of replicas + 1), * to wait for the desired amount of shard copies to become active before returning. * Index creation will only wait up until the timeout value for the number of shard copies - * to be active before returning. Check {@link ShrinkResponse#isShardsAcked()} to + * to be active before returning. Check {@link ResizeResponse#isShardsAcked()} to * determine if the requisite shard copies were all started before returning or timing out. * * @param waitForActiveShards number of active shard copies to wait on */ - public ShrinkRequestBuilder setWaitForActiveShards(ActiveShardCount waitForActiveShards) { + public ResizeRequestBuilder setWaitForActiveShards(ActiveShardCount waitForActiveShards) { this.request.setWaitForActiveShards(waitForActiveShards); return this; } @@ -70,7 +71,12 @@ public ShrinkRequestBuilder setWaitForActiveShards(ActiveShardCount waitForActiv * shard count is passed in, instead of having to first call {@link ActiveShardCount#from(int)} * to get the ActiveShardCount. */ - public ShrinkRequestBuilder setWaitForActiveShards(final int waitForActiveShards) { + public ResizeRequestBuilder setWaitForActiveShards(final int waitForActiveShards) { return setWaitForActiveShards(ActiveShardCount.from(waitForActiveShards)); } + + public ResizeRequestBuilder setResizeType(ResizeType type) { + this.request.setResizeType(type); + return this; + } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeResponse.java similarity index 86% rename from core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkResponse.java rename to core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeResponse.java index 0c5149f6bf353..cea74ced69cfc 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkResponse.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeResponse.java @@ -21,11 +21,11 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; -public final class ShrinkResponse extends CreateIndexResponse { - ShrinkResponse() { +public final class ResizeResponse extends CreateIndexResponse { + ResizeResponse() { } - ShrinkResponse(boolean acknowledged, boolean shardsAcked, String index) { + ResizeResponse(boolean acknowledged, boolean shardsAcked, String index) { super(acknowledged, shardsAcked, index); } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeType.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeType.java new file mode 100644 index 0000000000000..bca386a9567d6 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeType.java @@ -0,0 +1,27 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.shrink; + +/** + * The type of the resize operation + */ +public enum ResizeType { + SHRINK, SPLIT; +} diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkAction.java index 8b5b4670e3c4d..48c23d643ba4c 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkAction.java @@ -22,7 +22,7 @@ import org.elasticsearch.action.Action; import org.elasticsearch.client.ElasticsearchClient; -public class ShrinkAction extends Action { +public class ShrinkAction extends Action { public static final ShrinkAction INSTANCE = new ShrinkAction(); public static final String NAME = "indices:admin/shrink"; @@ -32,12 +32,12 @@ private ShrinkAction() { } @Override - public ShrinkResponse newResponse() { - return new ShrinkResponse(); + public ResizeResponse newResponse() { + return new ResizeResponse(); } @Override - public ShrinkRequestBuilder newRequestBuilder(ElasticsearchClient client) { - return new ShrinkRequestBuilder(client, this); + public ResizeRequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new ResizeRequestBuilder(client, this); } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java new file mode 100644 index 0000000000000..87dd9f9fa2d21 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java @@ -0,0 +1,201 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.shrink; + +import org.apache.lucene.index.IndexWriter; +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.stats.IndexShardStats; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.shard.DocsStats; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.util.Locale; +import java.util.Objects; +import java.util.Set; +import java.util.function.IntFunction; + +/** + * Main class to initiate resizing (shrink / split) an index into a new index + */ +public class TransportResizeAction extends TransportMasterNodeAction { + private final MetaDataCreateIndexService createIndexService; + private final Client client; + + @Inject + public TransportResizeAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, MetaDataCreateIndexService createIndexService, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, Client client) { + this(settings, ResizeAction.NAME, transportService, clusterService, threadPool, createIndexService, actionFilters, + indexNameExpressionResolver, client); + } + + protected TransportResizeAction(Settings settings, String actionName, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, MetaDataCreateIndexService createIndexService, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, Client client) { + super(settings, actionName, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, + ResizeRequest::new); + this.createIndexService = createIndexService; + this.client = client; + } + + + @Override + protected String executor() { + // we go async right away + return ThreadPool.Names.SAME; + } + + @Override + protected ResizeResponse newResponse() { + return new ResizeResponse(); + } + + @Override + protected ClusterBlockException checkBlock(ResizeRequest request, ClusterState state) { + return state.blocks().indexBlockedException(ClusterBlockLevel.METADATA_WRITE, request.getTargetIndexRequest().index()); + } + + @Override + protected void masterOperation(final ResizeRequest resizeRequest, final ClusterState state, + final ActionListener listener) { + + // there is no need to fetch docs stats for split but we keep it simple and do it anyway for simplicity of the code + final String sourceIndex = indexNameExpressionResolver.resolveDateMathExpression(resizeRequest.getSourceIndex()); + final String targetIndex = indexNameExpressionResolver.resolveDateMathExpression(resizeRequest.getTargetIndexRequest().index()); + client.admin().indices().prepareStats(sourceIndex).clear().setDocs(true).execute(new ActionListener() { + @Override + public void onResponse(IndicesStatsResponse indicesStatsResponse) { + CreateIndexClusterStateUpdateRequest updateRequest = prepareCreateIndexRequest(resizeRequest, state, + (i) -> { + IndexShardStats shard = indicesStatsResponse.getIndex(sourceIndex).getIndexShards().get(i); + return shard == null ? null : shard.getPrimary().getDocs(); + }, sourceIndex, targetIndex); + createIndexService.createIndex( + updateRequest, + ActionListener.wrap(response -> + listener.onResponse(new ResizeResponse(response.isAcknowledged(), response.isShardsAcked(), + updateRequest.index())), listener::onFailure + ) + ); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + + } + + // static for unittesting this method + static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final ResizeRequest resizeRequest, final ClusterState state + , final IntFunction perShardDocStats, String sourceIndexName, String targetIndexName) { + final CreateIndexRequest targetIndex = resizeRequest.getTargetIndexRequest(); + final IndexMetaData metaData = state.metaData().index(sourceIndexName); + if (metaData == null) { + throw new IndexNotFoundException(sourceIndexName); + } + final Settings targetIndexSettings = Settings.builder().put(targetIndex.settings()) + .normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX).build(); + final int numShards; + if (IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.exists(targetIndexSettings)) { + numShards = IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings); + } else { + assert resizeRequest.getResizeType() == ResizeType.SHRINK : "split must specify the number of shards explicitly"; + numShards = 1; + } + + for (int i = 0; i < numShards; i++) { + if (resizeRequest.getResizeType() == ResizeType.SHRINK) { + Set shardIds = IndexMetaData.selectShrinkShards(i, metaData, numShards); + long count = 0; + for (ShardId id : shardIds) { + DocsStats docsStats = perShardDocStats.apply(id.id()); + if (docsStats != null) { + count += docsStats.getCount(); + } + if (count > IndexWriter.MAX_DOCS) { + throw new IllegalStateException("Can't merge index with more than [" + IndexWriter.MAX_DOCS + + "] docs - too many documents in shards " + shardIds); + } + } + } else { + Objects.requireNonNull(IndexMetaData.selectSplitShard(i, metaData, numShards)); + // we just execute this to ensure we get the right exceptions if the number of shards is wrong or less then etc. + } + } + + if (IndexMetaData.INDEX_ROUTING_PARTITION_SIZE_SETTING.exists(targetIndexSettings)) { + throw new IllegalArgumentException("cannot provide a routing partition size value when resizing an index"); + } + if (IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.exists(targetIndexSettings)) { + throw new IllegalArgumentException("cannot provide index.number_of_routing_shards on resize"); + } + String cause = resizeRequest.getResizeType().name().toLowerCase(Locale.ROOT) + "_index"; + targetIndex.cause(cause); + Settings.Builder settingsBuilder = Settings.builder().put(targetIndexSettings); + settingsBuilder.put("index.number_of_shards", numShards); + targetIndex.settings(settingsBuilder); + + return new CreateIndexClusterStateUpdateRequest(targetIndex, + cause, targetIndex.index(), targetIndexName, true) + // mappings are updated on the node when creating in the shards, this prevents race-conditions since all mapping must be + // applied once we took the snapshot and if somebody messes things up and switches the index read/write and adds docs we miss + // the mappings for everything is corrupted and hard to debug + .ackTimeout(targetIndex.timeout()) + .masterNodeTimeout(targetIndex.masterNodeTimeout()) + .settings(targetIndex.settings()) + .aliases(targetIndex.aliases()) + .customs(targetIndex.customs()) + .waitForActiveShards(targetIndex.waitForActiveShards()) + .recoverFrom(metaData.getIndex()) + .resizeType(resizeRequest.getResizeType()); + } + + @Override + protected String getMasterActionName(DiscoveryNode node) { + if (node.getVersion().onOrAfter(ResizeAction.COMPATIBILITY_VERSION)){ + return super.getMasterActionName(node); + } else { + // this is for BWC - when we send this to version that doesn't have ResizeAction.NAME registered + // we have to send to shrink instead. + return ShrinkAction.NAME; + } + } +} diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkAction.java index 2555299709cda..acc88251970f3 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkAction.java @@ -19,143 +19,28 @@ package org.elasticsearch.action.admin.indices.shrink; -import org.apache.lucene.index.IndexWriter; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; -import org.elasticsearch.action.admin.indices.stats.IndexShardStats; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.master.TransportMasterNodeAction; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.block.ClusterBlockException; -import org.elasticsearch.cluster.block.ClusterBlockLevel; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.shard.DocsStats; -import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import java.util.Set; -import java.util.function.IntFunction; - /** - * Main class to initiate shrinking an index into a new index with a single shard + * Main class to initiate shrinking an index into a new index + * This class is only here for backwards compatibility. It will be replaced by + * TransportResizeAction in 7.x once this is backported */ -public class TransportShrinkAction extends TransportMasterNodeAction { - - private final MetaDataCreateIndexService createIndexService; - private final Client client; +public class TransportShrinkAction extends TransportResizeAction { @Inject public TransportShrinkAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool, MetaDataCreateIndexService createIndexService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, Client client) { - super(settings, ShrinkAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, - ShrinkRequest::new); - this.createIndexService = createIndexService; - this.client = client; + super(settings, ShrinkAction.NAME, transportService, clusterService, threadPool, createIndexService, actionFilters, + indexNameExpressionResolver, client); } - - @Override - protected String executor() { - // we go async right away - return ThreadPool.Names.SAME; - } - - @Override - protected ShrinkResponse newResponse() { - return new ShrinkResponse(); - } - - @Override - protected ClusterBlockException checkBlock(ShrinkRequest request, ClusterState state) { - return state.blocks().indexBlockedException(ClusterBlockLevel.METADATA_WRITE, request.getShrinkIndexRequest().index()); - } - - @Override - protected void masterOperation(final ShrinkRequest shrinkRequest, final ClusterState state, - final ActionListener listener) { - final String sourceIndex = indexNameExpressionResolver.resolveDateMathExpression(shrinkRequest.getSourceIndex()); - client.admin().indices().prepareStats(sourceIndex).clear().setDocs(true).execute(new ActionListener() { - @Override - public void onResponse(IndicesStatsResponse indicesStatsResponse) { - CreateIndexClusterStateUpdateRequest updateRequest = prepareCreateIndexRequest(shrinkRequest, state, - (i) -> { - IndexShardStats shard = indicesStatsResponse.getIndex(sourceIndex).getIndexShards().get(i); - return shard == null ? null : shard.getPrimary().getDocs(); - }, indexNameExpressionResolver); - createIndexService.createIndex( - updateRequest, - ActionListener.wrap(response -> - listener.onResponse(new ShrinkResponse(response.isAcknowledged(), response.isShardsAcked(), updateRequest.index())), - listener::onFailure - ) - ); - } - - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - }); - - } - - // static for unittesting this method - static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final ShrinkRequest shrinkRequest, final ClusterState state - , final IntFunction perShardDocStats, IndexNameExpressionResolver indexNameExpressionResolver) { - final String sourceIndex = indexNameExpressionResolver.resolveDateMathExpression(shrinkRequest.getSourceIndex()); - final CreateIndexRequest targetIndex = shrinkRequest.getShrinkIndexRequest(); - final String targetIndexName = indexNameExpressionResolver.resolveDateMathExpression(targetIndex.index()); - final IndexMetaData metaData = state.metaData().index(sourceIndex); - final Settings targetIndexSettings = Settings.builder().put(targetIndex.settings()) - .normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX).build(); - int numShards = 1; - if (IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.exists(targetIndexSettings)) { - numShards = IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings); - } - for (int i = 0; i < numShards; i++) { - Set shardIds = IndexMetaData.selectShrinkShards(i, metaData, numShards); - long count = 0; - for (ShardId id : shardIds) { - DocsStats docsStats = perShardDocStats.apply(id.id()); - if (docsStats != null) { - count += docsStats.getCount(); - } - if (count > IndexWriter.MAX_DOCS) { - throw new IllegalStateException("Can't merge index with more than [" + IndexWriter.MAX_DOCS - + "] docs - too many documents in shards " + shardIds); - } - } - - } - if (IndexMetaData.INDEX_ROUTING_PARTITION_SIZE_SETTING.exists(targetIndexSettings)) { - throw new IllegalArgumentException("cannot provide a routing partition size value when shrinking an index"); - } - targetIndex.cause("shrink_index"); - Settings.Builder settingsBuilder = Settings.builder().put(targetIndexSettings); - settingsBuilder.put("index.number_of_shards", numShards); - targetIndex.settings(settingsBuilder); - - return new CreateIndexClusterStateUpdateRequest(targetIndex, - "shrink_index", targetIndex.index(), targetIndexName, true) - // mappings are updated on the node when merging in the shards, this prevents race-conditions since all mapping must be - // applied once we took the snapshot and if somebody fucks things up and switches the index read/write and adds docs we miss - // the mappings for everything is corrupted and hard to debug - .ackTimeout(targetIndex.timeout()) - .masterNodeTimeout(targetIndex.masterNodeTimeout()) - .settings(targetIndex.settings()) - .aliases(targetIndex.aliases()) - .customs(targetIndex.customs()) - .waitForActiveShards(targetIndex.waitForActiveShards()) - .shrinkFrom(metaData.getIndex()); - } - } diff --git a/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java b/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java index 8cff2213c2111..feb47aa34fd86 100644 --- a/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java +++ b/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java @@ -32,6 +32,7 @@ import org.elasticsearch.cluster.NotMasterException; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.Writeable; @@ -189,7 +190,10 @@ protected void doRun() throws Exception { logger.debug("no known master node, scheduling a retry"); retry(null, masterChangePredicate); } else { - transportService.sendRequest(nodes.getMasterNode(), actionName, request, new ActionListenerResponseHandler(listener, TransportMasterNodeAction.this::newResponse) { + DiscoveryNode masterNode = nodes.getMasterNode(); + final String actionName = getMasterActionName(masterNode); + transportService.sendRequest(masterNode, actionName, request, new ActionListenerResponseHandler(listener, + TransportMasterNodeAction.this::newResponse) { @Override public void handleException(final TransportException exp) { Throwable cause = exp.unwrapCause(); @@ -229,4 +233,12 @@ public void onTimeout(TimeValue timeout) { ); } } + + /** + * Allows to conditionally return a different master node action name in the case an action gets renamed. + * This mainly for backwards compatibility should be used rarely + */ + protected String getMasterActionName(DiscoveryNode node) { + return actionName; + } } diff --git a/core/src/main/java/org/elasticsearch/client/IndicesAdminClient.java b/core/src/main/java/org/elasticsearch/client/IndicesAdminClient.java index b254039910c01..81de57f91afee 100644 --- a/core/src/main/java/org/elasticsearch/client/IndicesAdminClient.java +++ b/core/src/main/java/org/elasticsearch/client/IndicesAdminClient.java @@ -50,9 +50,6 @@ import org.elasticsearch.action.admin.indices.exists.types.TypesExistsRequest; import org.elasticsearch.action.admin.indices.exists.types.TypesExistsRequestBuilder; import org.elasticsearch.action.admin.indices.exists.types.TypesExistsResponse; -import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest; -import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequestBuilder; -import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; import org.elasticsearch.action.admin.indices.flush.FlushRequest; import org.elasticsearch.action.admin.indices.flush.FlushRequestBuilder; import org.elasticsearch.action.admin.indices.flush.FlushResponse; @@ -98,9 +95,9 @@ import org.elasticsearch.action.admin.indices.shards.IndicesShardStoreRequestBuilder; import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresRequest; import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse; -import org.elasticsearch.action.admin.indices.shrink.ShrinkRequest; -import org.elasticsearch.action.admin.indices.shrink.ShrinkRequestBuilder; -import org.elasticsearch.action.admin.indices.shrink.ShrinkResponse; +import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; +import org.elasticsearch.action.admin.indices.shrink.ResizeRequestBuilder; +import org.elasticsearch.action.admin.indices.shrink.ResizeResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest; import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; @@ -792,19 +789,19 @@ public interface IndicesAdminClient extends ElasticsearchClient { GetSettingsRequestBuilder prepareGetSettings(String... indices); /** - * Shrinks an index using an explicit request allowing to specify the settings, mappings and aliases of the target index of the index. + * Resize an index using an explicit request allowing to specify the settings, mappings and aliases of the target index of the index. */ - ShrinkRequestBuilder prepareShrinkIndex(String sourceIndex, String targetIndex); + ResizeRequestBuilder prepareResizeIndex(String sourceIndex, String targetIndex); /** - * Shrinks an index using an explicit request allowing to specify the settings, mappings and aliases of the target index of the index. + * Resize an index using an explicit request allowing to specify the settings, mappings and aliases of the target index of the index. */ - ActionFuture shrinkIndex(ShrinkRequest request); + ActionFuture resizeIndex(ResizeRequest request); /** * Shrinks an index using an explicit request allowing to specify the settings, mappings and aliases of the target index of the index. */ - void shrinkIndex(ShrinkRequest request, ActionListener listener); + void resizeIndex(ResizeRequest request, ActionListener listener); /** * Swaps the index pointed to by an alias given all provided conditions are satisfied diff --git a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java index c2b813d3d659e..c0da35a307981 100644 --- a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java +++ b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java @@ -232,10 +232,10 @@ import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresAction; import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresRequest; import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse; -import org.elasticsearch.action.admin.indices.shrink.ShrinkAction; -import org.elasticsearch.action.admin.indices.shrink.ShrinkRequest; -import org.elasticsearch.action.admin.indices.shrink.ShrinkRequestBuilder; -import org.elasticsearch.action.admin.indices.shrink.ShrinkResponse; +import org.elasticsearch.action.admin.indices.shrink.ResizeAction; +import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; +import org.elasticsearch.action.admin.indices.shrink.ResizeRequestBuilder; +import org.elasticsearch.action.admin.indices.shrink.ResizeResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction; import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest; import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder; @@ -1730,19 +1730,19 @@ public GetSettingsRequestBuilder prepareGetSettings(String... indices) { } @Override - public ShrinkRequestBuilder prepareShrinkIndex(String sourceIndex, String targetIndex) { - return new ShrinkRequestBuilder(this, ShrinkAction.INSTANCE).setSourceIndex(sourceIndex) + public ResizeRequestBuilder prepareResizeIndex(String sourceIndex, String targetIndex) { + return new ResizeRequestBuilder(this, ResizeAction.INSTANCE).setSourceIndex(sourceIndex) .setTargetIndex(new CreateIndexRequest(targetIndex)); } @Override - public ActionFuture shrinkIndex(ShrinkRequest request) { - return execute(ShrinkAction.INSTANCE, request); + public ActionFuture resizeIndex(ResizeRequest request) { + return execute(ResizeAction.INSTANCE, request); } @Override - public void shrinkIndex(ShrinkRequest request, ActionListener listener) { - execute(ShrinkAction.INSTANCE, request, listener); + public void resizeIndex(ResizeRequest request, ActionListener listener) { + execute(ResizeAction.INSTANCE, request, listener); } @Override diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportProxyClient.java b/core/src/main/java/org/elasticsearch/client/transport/TransportProxyClient.java index 5436bef172a47..e07fab0092d0e 100644 --- a/core/src/main/java/org/elasticsearch/client/transport/TransportProxyClient.java +++ b/core/src/main/java/org/elasticsearch/client/transport/TransportProxyClient.java @@ -56,6 +56,7 @@ final class TransportProxyClient { ActionRequestBuilder> void execute(final Action action, final Request request, ActionListener listener) { final TransportActionNodeProxy proxy = proxies.get(action); + assert proxy != null : "no proxy found for action: " + action; nodesService.execute((n, l) -> proxy.execute(n, request, l), listener); } } diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java index a5b7f422a9322..a4bb6a559254c 100644 --- a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java +++ b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java @@ -50,6 +50,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.NodeVersionAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.RebalanceOnlyWhenActiveAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.ReplicaAfterPrimaryActiveAllocationDecider; +import org.elasticsearch.cluster.routing.allocation.decider.ResizeAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.SnapshotInProgressAllocationDecider; @@ -182,6 +183,7 @@ public static Collection createAllocationDeciders(Settings se // collect deciders by class so that we can detect duplicates Map deciders = new LinkedHashMap<>(); addAllocationDecider(deciders, new MaxRetryAllocationDecider(settings)); + addAllocationDecider(deciders, new ResizeAllocationDecider(settings)); addAllocationDecider(deciders, new ReplicaAfterPrimaryActiveAllocationDecider(settings)); addAllocationDecider(deciders, new RebalanceOnlyWhenActiveAllocationDecider(settings)); addAllocationDecider(deciders, new ClusterRebalanceAllocationDecider(settings, clusterSettings)); diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 06f203595b313..3d14670e52771 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -65,6 +65,7 @@ import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.Locale; import java.util.Map; import java.util.Set; @@ -195,6 +196,24 @@ static Setting buildNumberOfShardsSetting() { public static final Setting INDEX_ROUTING_PARTITION_SIZE_SETTING = Setting.intSetting(SETTING_ROUTING_PARTITION_SIZE, 1, 1, Property.IndexScope); + public static final Setting INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING = + Setting.intSetting("index.number_of_routing_shards", INDEX_NUMBER_OF_SHARDS_SETTING, 1, new Setting.Validator() { + @Override + public void validate(Integer numRoutingShards, Map, Integer> settings) { + Integer numShards = settings.get(INDEX_NUMBER_OF_SHARDS_SETTING); + if (numRoutingShards < numShards) { + throw new IllegalArgumentException("index.number_of_routing_shards [" + numRoutingShards + + "] must be >= index.number_of_shards [" + numShards + "]"); + } + getRoutingFactor(numShards, numRoutingShards); + } + + @Override + public Iterator> settings() { + return Collections.singleton(INDEX_NUMBER_OF_SHARDS_SETTING).iterator(); + } + }, Property.IndexScope); + public static final String SETTING_AUTO_EXPAND_REPLICAS = "index.auto_expand_replicas"; public static final Setting INDEX_AUTO_EXPAND_REPLICAS_SETTING = AutoExpandReplicas.SETTING; public static final String SETTING_READ_ONLY = "index.blocks.read_only"; @@ -453,14 +472,22 @@ public MappingMetaData mapping(String mappingType) { return mappings.get(mappingType); } + // we keep the shrink settings for BWC - this can be removed in 8.0 + // we can't remove in 7 since this setting might be baked into an index coming in via a full cluster restart from 6.0 public static final String INDEX_SHRINK_SOURCE_UUID_KEY = "index.shrink.source.uuid"; public static final String INDEX_SHRINK_SOURCE_NAME_KEY = "index.shrink.source.name"; + public static final String INDEX_RESIZE_SOURCE_UUID_KEY = "index.resize.source.uuid"; + public static final String INDEX_RESIZE_SOURCE_NAME_KEY = "index.resize.source.name"; public static final Setting INDEX_SHRINK_SOURCE_UUID = Setting.simpleString(INDEX_SHRINK_SOURCE_UUID_KEY); public static final Setting INDEX_SHRINK_SOURCE_NAME = Setting.simpleString(INDEX_SHRINK_SOURCE_NAME_KEY); - - - public Index getMergeSourceIndex() { - return INDEX_SHRINK_SOURCE_UUID.exists(settings) ? new Index(INDEX_SHRINK_SOURCE_NAME.get(settings), INDEX_SHRINK_SOURCE_UUID.get(settings)) : null; + public static final Setting INDEX_RESIZE_SOURCE_UUID = Setting.simpleString(INDEX_RESIZE_SOURCE_UUID_KEY, + INDEX_SHRINK_SOURCE_UUID); + public static final Setting INDEX_RESIZE_SOURCE_NAME = Setting.simpleString(INDEX_RESIZE_SOURCE_NAME_KEY, + INDEX_SHRINK_SOURCE_NAME); + + public Index getResizeSourceIndex() { + return INDEX_RESIZE_SOURCE_UUID.exists(settings) || INDEX_SHRINK_SOURCE_UUID.exists(settings) + ? new Index(INDEX_RESIZE_SOURCE_NAME.get(settings), INDEX_RESIZE_SOURCE_UUID.get(settings)) : null; } /** @@ -1006,7 +1033,6 @@ public IndexMetaData build() { throw new IllegalArgumentException("routing partition size [" + routingPartitionSize + "] should be a positive number" + " less than the number of shards [" + getRoutingNumShards() + "] for [" + index + "]"); } - // fill missing slots in inSyncAllocationIds with empty set if needed and make all entries immutable ImmutableOpenIntMap.Builder> filledInSyncAllocationIds = ImmutableOpenIntMap.builder(); for (int i = 0; i < numberOfShards; i++) { @@ -1293,12 +1319,50 @@ public int getRoutingNumShards() { /** * Returns the routing factor for this index. The default is 1. * - * @see #getRoutingFactor(IndexMetaData, int) for details + * @see #getRoutingFactor(int, int) for details */ public int getRoutingFactor() { return routingFactor; } + /** + * Returns the source shard ID to split the given target shard off + * @param shardId the id of the target shard to split into + * @param sourceIndexMetadata the source index metadata + * @param numTargetShards the total number of shards in the target index + * @return a the source shard ID to split off from + */ + public static ShardId selectSplitShard(int shardId, IndexMetaData sourceIndexMetadata, int numTargetShards) { + if (shardId >= numTargetShards) { + throw new IllegalArgumentException("the number of target shards (" + numTargetShards + ") must be greater than the shard id: " + + shardId); + } + int numSourceShards = sourceIndexMetadata.getNumberOfShards(); + if (numSourceShards > numTargetShards) { + throw new IllegalArgumentException("the number of source shards [" + numSourceShards + + "] must be less that the number of target shards [" + numTargetShards + "]"); + } + int routingFactor = getRoutingFactor(numSourceShards, numTargetShards); + // this is just an additional assertion that ensures we are a factor of the routing num shards. + assert getRoutingFactor(numTargetShards, sourceIndexMetadata.getRoutingNumShards()) >= 0; + return new ShardId(sourceIndexMetadata.getIndex(), shardId/routingFactor); + } + + /** + * Selects the source shards for a local shard recovery. This might either be a split or a shrink operation. + * @param shardId the target shard ID to select the source shards for + * @param sourceIndexMetadata the source metadata + * @param numTargetShards the number of target shards + */ + public static Set selectRecoverFromShards(int shardId, IndexMetaData sourceIndexMetadata, int numTargetShards) { + if (sourceIndexMetadata.getNumberOfShards() > numTargetShards) { + return selectShrinkShards(shardId, sourceIndexMetadata, numTargetShards); + } else if (sourceIndexMetadata.getNumberOfShards() < numTargetShards) { + return Collections.singleton(selectSplitShard(shardId, sourceIndexMetadata, numTargetShards)); + } + throw new IllegalArgumentException("can't select recover from shards if both indices have the same number of shards"); + } + /** * Returns the source shard ids to shrink into the given shard id. * @param shardId the id of the target shard to shrink to @@ -1311,7 +1375,11 @@ public static Set selectShrinkShards(int shardId, IndexMetaData sourceI throw new IllegalArgumentException("the number of target shards (" + numTargetShards + ") must be greater than the shard id: " + shardId); } - int routingFactor = getRoutingFactor(sourceIndexMetadata, numTargetShards); + if (sourceIndexMetadata.getNumberOfShards() < numTargetShards) { + throw new IllegalArgumentException("the number of target shards [" + numTargetShards + +"] must be less that the number of source shards [" + sourceIndexMetadata.getNumberOfShards() + "]"); + } + int routingFactor = getRoutingFactor(sourceIndexMetadata.getNumberOfShards(), numTargetShards); Set shards = new HashSet<>(routingFactor); for (int i = shardId * routingFactor; i < routingFactor*shardId + routingFactor; i++) { shards.add(new ShardId(sourceIndexMetadata.getIndex(), i)); @@ -1325,21 +1393,30 @@ public static Set selectShrinkShards(int shardId, IndexMetaData sourceI * {@link org.elasticsearch.cluster.routing.OperationRouting#generateShardId(IndexMetaData, String, String)} to guarantee consistent * hashing / routing of documents even if the number of shards changed (ie. a shrunk index). * - * @param sourceIndexMetadata the metadata of the source index + * @param sourceNumberOfShards the total number of shards in the source index * @param targetNumberOfShards the total number of shards in the target index * @return the routing factor for and shrunk index with the given number of target shards. * @throws IllegalArgumentException if the number of source shards is less than the number of target shards or if the source shards * are not divisible by the number of target shards. */ - public static int getRoutingFactor(IndexMetaData sourceIndexMetadata, int targetNumberOfShards) { - int sourceNumberOfShards = sourceIndexMetadata.getNumberOfShards(); - if (sourceNumberOfShards < targetNumberOfShards) { - throw new IllegalArgumentException("the number of target shards must be less that the number of source shards"); - } - int factor = sourceNumberOfShards / targetNumberOfShards; - if (factor * targetNumberOfShards != sourceNumberOfShards || factor <= 1) { - throw new IllegalArgumentException("the number of source shards [" + sourceNumberOfShards + "] must be a must be a multiple of [" - + targetNumberOfShards + "]"); + public static int getRoutingFactor(int sourceNumberOfShards, int targetNumberOfShards) { + final int factor; + if (sourceNumberOfShards < targetNumberOfShards) { // split + factor = targetNumberOfShards / sourceNumberOfShards; + if (factor * sourceNumberOfShards != targetNumberOfShards || factor <= 1) { + throw new IllegalArgumentException("the number of source shards [" + sourceNumberOfShards + "] must be a must be a " + + "factor of [" + + targetNumberOfShards + "]"); + } + } else if (sourceNumberOfShards > targetNumberOfShards) { // shrink + factor = sourceNumberOfShards / targetNumberOfShards; + if (factor * targetNumberOfShards != sourceNumberOfShards || factor <= 1) { + throw new IllegalArgumentException("the number of source shards [" + sourceNumberOfShards + "] must be a must be a " + + "multiple of [" + + targetNumberOfShards + "]"); + } + } else { + factor = 1; } return factor; } diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 643987862ff2f..49568ab300f03 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -31,6 +31,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; +import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.ActiveShardsObserver; import org.elasticsearch.cluster.AckedClusterStateUpdateTask; @@ -116,7 +117,6 @@ public class MetaDataCreateIndexService extends AbstractComponent { private final IndexScopedSettings indexScopedSettings; private final ActiveShardsObserver activeShardsObserver; private final NamedXContentRegistry xContentRegistry; - private final ThreadPool threadPool; @Inject public MetaDataCreateIndexService(Settings settings, ClusterService clusterService, @@ -132,7 +132,6 @@ public MetaDataCreateIndexService(Settings settings, ClusterService clusterServi this.env = env; this.indexScopedSettings = indexScopedSettings; this.activeShardsObserver = new ActiveShardsObserver(settings, clusterService, threadPool); - this.threadPool = threadPool; this.xContentRegistry = xContentRegistry; } @@ -298,9 +297,9 @@ public ClusterState execute(ClusterState currentState) throws Exception { customs.put(entry.getKey(), entry.getValue()); } - final Index shrinkFromIndex = request.shrinkFrom(); + final Index recoverFromIndex = request.recoverFrom(); - if (shrinkFromIndex == null) { + if (recoverFromIndex == null) { // apply templates, merging the mappings into the request mapping if exists for (IndexTemplateMetaData template : templates) { templateNames.add(template.getName()); @@ -351,7 +350,7 @@ public ClusterState execute(ClusterState currentState) throws Exception { } } Settings.Builder indexSettingsBuilder = Settings.builder(); - if (shrinkFromIndex == null) { + if (recoverFromIndex == null) { // apply templates, here, in reverse order, since first ones are better matching for (int i = templates.size() - 1; i >= 0; i--) { indexSettingsBuilder.put(templates.get(i).settings()); @@ -383,28 +382,34 @@ public ClusterState execute(ClusterState currentState) throws Exception { final IndexMetaData.Builder tmpImdBuilder = IndexMetaData.builder(request.index()); final int routingNumShards; - if (shrinkFromIndex == null) { - routingNumShards = IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(indexSettingsBuilder.build()); + if (recoverFromIndex == null) { + Settings idxSettings = indexSettingsBuilder.build(); + routingNumShards = IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(idxSettings); } else { - final IndexMetaData sourceMetaData = currentState.metaData().getIndexSafe(shrinkFromIndex); + assert IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.exists(indexSettingsBuilder.build()) == false + : "index.number_of_routing_shards should be present on the target index on resize"; + final IndexMetaData sourceMetaData = currentState.metaData().getIndexSafe(recoverFromIndex); routingNumShards = sourceMetaData.getRoutingNumShards(); } + // remove the setting it's temporary and is only relevant once we create the index + indexSettingsBuilder.remove(IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.getKey()); tmpImdBuilder.setRoutingNumShards(routingNumShards); - if (shrinkFromIndex != null) { - prepareShrinkIndexSettings( - currentState, mappings.keySet(), indexSettingsBuilder, shrinkFromIndex, request.index()); + if (recoverFromIndex != null) { + assert request.resizeType() != null; + prepareResizeIndexSettings( + currentState, mappings.keySet(), indexSettingsBuilder, recoverFromIndex, request.index(), request.resizeType()); } final Settings actualIndexSettings = indexSettingsBuilder.build(); tmpImdBuilder.settings(actualIndexSettings); - if (shrinkFromIndex != null) { + if (recoverFromIndex != null) { /* * We need to arrange that the primary term on all the shards in the shrunken index is at least as large as * the maximum primary term on all the shards in the source index. This ensures that we have correct * document-level semantics regarding sequence numbers in the shrunken index. */ - final IndexMetaData sourceMetaData = currentState.metaData().getIndexSafe(shrinkFromIndex); + final IndexMetaData sourceMetaData = currentState.metaData().getIndexSafe(recoverFromIndex); final long primaryTerm = IntStream .range(0, sourceMetaData.getNumberOfShards()) @@ -439,7 +444,7 @@ public ClusterState execute(ClusterState currentState) throws Exception { throw e; } - if (request.shrinkFrom() == null) { + if (request.recoverFrom() == null) { // now that the mapping is merged we can validate the index sort. // we cannot validate for index shrinking since the mapping is empty // at this point. The validation will take place later in the process @@ -606,35 +611,14 @@ List getIndexSettingsValidationErrors(Settings settings) { static List validateShrinkIndex(ClusterState state, String sourceIndex, Set targetIndexMappingsTypes, String targetIndexName, Settings targetIndexSettings) { - if (state.metaData().hasIndex(targetIndexName)) { - throw new ResourceAlreadyExistsException(state.metaData().index(targetIndexName).getIndex()); - } - final IndexMetaData sourceMetaData = state.metaData().index(sourceIndex); - if (sourceMetaData == null) { - throw new IndexNotFoundException(sourceIndex); - } - // ensure index is read-only - if (state.blocks().indexBlocked(ClusterBlockLevel.WRITE, sourceIndex) == false) { - throw new IllegalStateException("index " + sourceIndex + " must be read-only to shrink index. use \"index.blocks.write=true\""); - } + IndexMetaData sourceMetaData = validateResize(state, sourceIndex, targetIndexMappingsTypes, targetIndexName, targetIndexSettings); + assert IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.exists(targetIndexSettings); + IndexMetaData.selectShrinkShards(0, sourceMetaData, IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings)); if (sourceMetaData.getNumberOfShards() == 1) { throw new IllegalArgumentException("can't shrink an index with only one shard"); } - - if ((targetIndexMappingsTypes.size() > 1 || - (targetIndexMappingsTypes.isEmpty() || targetIndexMappingsTypes.contains(MapperService.DEFAULT_MAPPING)) == false)) { - throw new IllegalArgumentException("mappings are not allowed when shrinking indices" + - ", all mappings are copied from the source index"); - } - - if (IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.exists(targetIndexSettings)) { - // this method applies all necessary checks ie. if the target shards are less than the source shards - // of if the source shards are divisible by the number of target shards - IndexMetaData.getRoutingFactor(sourceMetaData, IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings)); - } - // now check that index is all on one node final IndexRoutingTable table = state.routingTable().index(sourceIndex); Map nodesToNumRouting = new HashMap<>(); @@ -657,27 +641,82 @@ static List validateShrinkIndex(ClusterState state, String sourceIndex, return nodesToAllocateOn; } - static void prepareShrinkIndexSettings(ClusterState currentState, Set mappingKeys, Settings.Builder indexSettingsBuilder, Index shrinkFromIndex, String shrinkIntoName) { - final IndexMetaData sourceMetaData = currentState.metaData().index(shrinkFromIndex.getName()); + static void validateSplitIndex(ClusterState state, String sourceIndex, + Set targetIndexMappingsTypes, String targetIndexName, + Settings targetIndexSettings) { + IndexMetaData sourceMetaData = validateResize(state, sourceIndex, targetIndexMappingsTypes, targetIndexName, targetIndexSettings); + IndexMetaData.selectSplitShard(0, sourceMetaData, IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings)); + if (sourceMetaData.getCreationVersion().before(Version.V_6_0_0_alpha1)) { + // ensure we have a single type since this would make the splitting code considerably more complex + // and a 5.x index would not be splittable unless it has been shrunk before so rather opt out of the complexity + // since in 5.x we don't have a setting to artificially set the number of routing shards + throw new IllegalStateException("source index created version is too old to apply a split operation"); + } + + } + + static IndexMetaData validateResize(ClusterState state, String sourceIndex, + Set targetIndexMappingsTypes, String targetIndexName, + Settings targetIndexSettings) { + if (state.metaData().hasIndex(targetIndexName)) { + throw new ResourceAlreadyExistsException(state.metaData().index(targetIndexName).getIndex()); + } + final IndexMetaData sourceMetaData = state.metaData().index(sourceIndex); + if (sourceMetaData == null) { + throw new IndexNotFoundException(sourceIndex); + } + // ensure index is read-only + if (state.blocks().indexBlocked(ClusterBlockLevel.WRITE, sourceIndex) == false) { + throw new IllegalStateException("index " + sourceIndex + " must be read-only to resize index. use \"index.blocks.write=true\""); + } + + if ((targetIndexMappingsTypes.size() > 1 || + (targetIndexMappingsTypes.isEmpty() || targetIndexMappingsTypes.contains(MapperService.DEFAULT_MAPPING)) == false)) { + throw new IllegalArgumentException("mappings are not allowed when resizing indices" + + ", all mappings are copied from the source index"); + } + + if (IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.exists(targetIndexSettings)) { + // this method applies all necessary checks ie. if the target shards are less than the source shards + // of if the source shards are divisible by the number of target shards + IndexMetaData.getRoutingFactor(sourceMetaData.getNumberOfShards(), + IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings)); + } + return sourceMetaData; + } + + static void prepareResizeIndexSettings(ClusterState currentState, Set mappingKeys, Settings.Builder indexSettingsBuilder, + Index resizeSourceIndex, String resizeIntoName, ResizeType type) { + final IndexMetaData sourceMetaData = currentState.metaData().index(resizeSourceIndex.getName()); + if (type == ResizeType.SHRINK) { + final List nodesToAllocateOn = validateShrinkIndex(currentState, resizeSourceIndex.getName(), + mappingKeys, resizeIntoName, indexSettingsBuilder.build()); + indexSettingsBuilder + // we use "i.r.a.initial_recovery" rather than "i.r.a.require|include" since we want the replica to allocate right away + // once we are allocated. + .put(IndexMetaData.INDEX_ROUTING_INITIAL_RECOVERY_GROUP_SETTING.getKey() + "_id", + Strings.arrayToCommaDelimitedString(nodesToAllocateOn.toArray())) + // we only try once and then give up with a shrink index + .put("index.allocation.max_retries", 1) + // we add the legacy way of specifying it here for BWC. We can remove this once it's backported to 6.x + .put(IndexMetaData.INDEX_SHRINK_SOURCE_NAME.getKey(), resizeSourceIndex.getName()) + .put(IndexMetaData.INDEX_SHRINK_SOURCE_UUID.getKey(), resizeSourceIndex.getUUID()); + } else if (type == ResizeType.SPLIT) { + validateSplitIndex(currentState, resizeSourceIndex.getName(), mappingKeys, resizeIntoName, indexSettingsBuilder.build()); + } else { + throw new IllegalStateException("unknown resize type is " + type); + } - final List nodesToAllocateOn = validateShrinkIndex(currentState, shrinkFromIndex.getName(), - mappingKeys, shrinkIntoName, indexSettingsBuilder.build()); final Predicate sourceSettingsPredicate = (s) -> s.startsWith("index.similarity.") || s.startsWith("index.analysis.") || s.startsWith("index.sort."); indexSettingsBuilder - // we use "i.r.a.initial_recovery" rather than "i.r.a.require|include" since we want the replica to allocate right away - // once we are allocated. - .put(IndexMetaData.INDEX_ROUTING_INITIAL_RECOVERY_GROUP_SETTING.getKey() + "_id", - Strings.arrayToCommaDelimitedString(nodesToAllocateOn.toArray())) - // we only try once and then give up with a shrink index - .put("index.allocation.max_retries", 1) // now copy all similarity / analysis / sort settings - this overrides all settings from the user unless they // wanna add extra settings .put(IndexMetaData.SETTING_VERSION_CREATED, sourceMetaData.getCreationVersion()) .put(IndexMetaData.SETTING_VERSION_UPGRADED, sourceMetaData.getUpgradedVersion()) .put(sourceMetaData.getSettings().filter(sourceSettingsPredicate)) .put(IndexMetaData.SETTING_ROUTING_PARTITION_SIZE, sourceMetaData.getRoutingPartitionSize()) - .put(IndexMetaData.INDEX_SHRINK_SOURCE_NAME.getKey(), shrinkFromIndex.getName()) - .put(IndexMetaData.INDEX_SHRINK_SOURCE_UUID.getKey(), shrinkFromIndex.getUUID()); + .put(IndexMetaData.INDEX_RESIZE_SOURCE_NAME.getKey(), resizeSourceIndex.getName()) + .put(IndexMetaData.INDEX_RESIZE_SOURCE_UUID.getKey(), resizeSourceIndex.getUUID()); } } diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java index 5a0bd0d426313..5a4e0c78414dd 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java @@ -411,7 +411,7 @@ private Builder initializeEmpty(IndexMetaData indexMetaData, UnassignedInfo unas if (indexMetaData.inSyncAllocationIds(shardNumber).isEmpty() == false) { // we have previous valid copies for this shard. use them for recovery primaryRecoverySource = StoreRecoverySource.EXISTING_STORE_INSTANCE; - } else if (indexMetaData.getMergeSourceIndex() != null) { + } else if (indexMetaData.getResizeSourceIndex() != null) { // this is a new index but the initial shards should merged from another index primaryRecoverySource = LocalShardsRecoverySource.INSTANCE; } else { diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java index 87adb55704a25..005600ceb4431 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java @@ -259,7 +259,7 @@ public ShardId shardId(ClusterState clusterState, String index, String id, @Null return new ShardId(indexMetaData.getIndex(), generateShardId(indexMetaData, id, routing)); } - static int generateShardId(IndexMetaData indexMetaData, @Nullable String id, @Nullable String routing) { + public static int generateShardId(IndexMetaData indexMetaData, @Nullable String id, @Nullable String routing) { final String effectiveRouting; final int partitionOffset; diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java index 56663be1ef427..2a323af5f8435 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java @@ -403,14 +403,14 @@ private Decision earlyTerminate(RoutingAllocation allocation, ImmutableOpenMap shardIds = IndexMetaData.selectShrinkShards(shard.id(), sourceIndexMeta, metaData.getNumberOfShards()); + final Set shardIds = IndexMetaData.selectRecoverFromShards(shard.id(), sourceIndexMeta, metaData.getNumberOfShards()); for (IndexShardRoutingTable shardRoutingTable : allocation.routingTable().index(mergeSourceIndex.getName())) { if (shardIds.contains(shardRoutingTable.shardId())) { targetShardSize += info.getShardSize(shardRoutingTable.primaryShard(), 0); diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ResizeAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ResizeAllocationDecider.java new file mode 100644 index 0000000000000..a0ebf7ddba923 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ResizeAllocationDecider.java @@ -0,0 +1,102 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster.routing.allocation.decider; + +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.indices.shrink.ResizeAction; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.routing.RecoverySource; +import org.elasticsearch.cluster.routing.RoutingNode; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.UnassignedInfo; +import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.shard.ShardId; + +import java.util.Set; + +/** + * An allocation decider that ensures we allocate the shards of a target index for resize operations next to the source primaries + */ +public class ResizeAllocationDecider extends AllocationDecider { + + public static final String NAME = "resize"; + + /** + * Initializes a new {@link ResizeAllocationDecider} + * + * @param settings {@link Settings} used by this {@link AllocationDecider} + */ + public ResizeAllocationDecider(Settings settings) { + super(settings); + } + + @Override + public Decision canAllocate(ShardRouting shardRouting, RoutingAllocation allocation) { + return canAllocate(shardRouting, null, allocation); + } + + @Override + public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + final UnassignedInfo unassignedInfo = shardRouting.unassignedInfo(); + if (unassignedInfo != null && shardRouting.recoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS) { + // we only make decisions here if we have an unassigned info and we have to recover from another index ie. split / shrink + final IndexMetaData indexMetaData = allocation.metaData().getIndexSafe(shardRouting.index()); + Index resizeSourceIndex = indexMetaData.getResizeSourceIndex(); + assert resizeSourceIndex != null; + if (allocation.metaData().index(resizeSourceIndex) == null) { + return allocation.decision(Decision.NO, NAME, "resize source index [%s] doesn't exists", resizeSourceIndex.toString()); + } + IndexMetaData sourceIndexMetaData = allocation.metaData().getIndexSafe(resizeSourceIndex); + if (indexMetaData.getNumberOfShards() < sourceIndexMetaData.getNumberOfShards()) { + // this only handles splits so far. + return Decision.ALWAYS; + } + + ShardId shardId = IndexMetaData.selectSplitShard(shardRouting.id(), sourceIndexMetaData, indexMetaData.getNumberOfShards()); + ShardRouting sourceShardRouting = allocation.routingNodes().activePrimary(shardId); + if (sourceShardRouting == null) { + return allocation.decision(Decision.NO, NAME, "source primary shard [%s] is not active", shardId); + } + if (node != null) { // we might get called from the 2 param canAllocate method.. + if (node.node().getVersion().before(ResizeAction.COMPATIBILITY_VERSION)) { + return allocation.decision(Decision.NO, NAME, "node [%s] is too old to split a shard", node.nodeId()); + } + if (sourceShardRouting.currentNodeId().equals(node.nodeId())) { + return allocation.decision(Decision.YES, NAME, "source primary is allocated on this node"); + } else { + return allocation.decision(Decision.NO, NAME, "source primary is allocated on another node"); + } + } else { + return allocation.decision(Decision.YES, NAME, "source primary is active"); + } + } + return super.canAllocate(shardRouting, node, allocation); + } + + @Override + public Decision canForceAllocatePrimary(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + assert shardRouting.primary() : "must not call canForceAllocatePrimary on a non-primary shard " + shardRouting; + return canAllocate(shardRouting, node, allocation); + } +} diff --git a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index 235300b8267f6..b37fbb0dce65c 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -70,6 +70,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING, IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING, IndexMetaData.INDEX_ROUTING_PARTITION_SIZE_SETTING, + IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING, IndexMetaData.INDEX_READ_ONLY_SETTING, IndexMetaData.INDEX_BLOCKS_READ_SETTING, IndexMetaData.INDEX_BLOCKS_WRITE_SETTING, @@ -197,6 +198,8 @@ protected boolean isPrivateSetting(String key) { case MergePolicyConfig.INDEX_MERGE_ENABLED: case IndexMetaData.INDEX_SHRINK_SOURCE_UUID_KEY: case IndexMetaData.INDEX_SHRINK_SOURCE_NAME_KEY: + case IndexMetaData.INDEX_RESIZE_SOURCE_UUID_KEY: + case IndexMetaData.INDEX_RESIZE_SOURCE_NAME_KEY: case IndexSettings.INDEX_MAPPING_SINGLE_TYPE_SETTING_KEY: // this was settable in 5.x but not anymore in 6.x so we have to preserve the value ie. make it read-only // this can be removed in later versions diff --git a/core/src/main/java/org/elasticsearch/common/settings/Setting.java b/core/src/main/java/org/elasticsearch/common/settings/Setting.java index f35df27e3b338..9b99e67c8c4da 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/Setting.java +++ b/core/src/main/java/org/elasticsearch/common/settings/Setting.java @@ -51,6 +51,7 @@ import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.IntConsumer; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -907,6 +908,12 @@ public static Setting intSetting(String key, Setting fallbackS return new Setting<>(key, fallbackSetting, (s) -> parseInt(s, minValue, key), properties); } + public static Setting intSetting(String key, Setting fallbackSetting, int minValue, Validator validator, + Property... properties) { + return new Setting<>(new SimpleKey(key), fallbackSetting, fallbackSetting::getRaw, (s) -> parseInt(s, minValue, key),validator, + properties); + } + public static Setting longSetting(String key, long defaultValue, long minValue, Property... properties) { return new Setting<>(key, (s) -> Long.toString(defaultValue), (s) -> parseLong(s, minValue, key), properties); } @@ -915,6 +922,10 @@ public static Setting simpleString(String key, Property... properties) { return new Setting<>(key, s -> "", Function.identity(), properties); } + public static Setting simpleString(String key, Setting fallback, Property... properties) { + return new Setting<>(key, fallback, Function.identity(), properties); + } + public static Setting simpleString(String key, Validator validator, Property... properties) { return new Setting<>(new SimpleKey(key), null, s -> "", Function.identity(), validator, properties); } diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java index fc47c71573c1f..f2aab70e81920 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -139,6 +139,7 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.EnumSet; import java.util.List; import java.util.Locale; @@ -1996,25 +1997,32 @@ public void startRecovery(RecoveryState recoveryState, PeerRecoveryTargetService break; case LOCAL_SHARDS: final IndexMetaData indexMetaData = indexSettings().getIndexMetaData(); - final Index mergeSourceIndex = indexMetaData.getMergeSourceIndex(); + final Index resizeSourceIndex = indexMetaData.getResizeSourceIndex(); final List startedShards = new ArrayList<>(); - final IndexService sourceIndexService = indicesService.indexService(mergeSourceIndex); - final int numShards = sourceIndexService != null ? sourceIndexService.getIndexSettings().getNumberOfShards() : -1; + final IndexService sourceIndexService = indicesService.indexService(resizeSourceIndex); + final Set requiredShards; + final int numShards; if (sourceIndexService != null) { + requiredShards = IndexMetaData.selectRecoverFromShards(shardId().id(), + sourceIndexService.getMetaData(), indexMetaData.getNumberOfShards()); for (IndexShard shard : sourceIndexService) { - if (shard.state() == IndexShardState.STARTED) { + if (shard.state() == IndexShardState.STARTED && requiredShards.contains(shard.shardId())) { startedShards.add(shard); } } + numShards = requiredShards.size(); + } else { + numShards = -1; + requiredShards = Collections.emptySet(); } + if (numShards == startedShards.size()) { + assert requiredShards.isEmpty() == false; markAsRecovering("from local shards", recoveryState); // mark the shard as recovering on the cluster state thread threadPool.generic().execute(() -> { try { - final Set shards = IndexMetaData.selectShrinkShards(shardId().id(), sourceIndexService.getMetaData(), - +indexMetaData.getNumberOfShards()); if (recoverFromLocalShards(mappingUpdateConsumer, startedShards.stream() - .filter((s) -> shards.contains(s.shardId())).collect(Collectors.toList()))) { + .filter((s) -> requiredShards.contains(s.shardId())).collect(Collectors.toList()))) { recoveryListener.onRecoveryDone(recoveryState); } } catch (Exception e) { @@ -2025,9 +2033,9 @@ public void startRecovery(RecoveryState recoveryState, PeerRecoveryTargetService } else { final RuntimeException e; if (numShards == -1) { - e = new IndexNotFoundException(mergeSourceIndex); + e = new IndexNotFoundException(resizeSourceIndex); } else { - e = new IllegalStateException("not all shards from index " + mergeSourceIndex + e = new IllegalStateException("not all required shards of index " + resizeSourceIndex + " are started yet, expected " + numShards + " found " + startedShards.size() + " can't recover shard " + shardId()); } diff --git a/core/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java b/core/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java new file mode 100644 index 0000000000000..94aee085175a0 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java @@ -0,0 +1,245 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.shard; + +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.PostingsEnum; +import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.search.Weight; +import org.apache.lucene.util.BitSetIterator; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.FixedBitSet; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.routing.OperationRouting; +import org.elasticsearch.index.mapper.IdFieldMapper; +import org.elasticsearch.index.mapper.RoutingFieldMapper; +import org.elasticsearch.index.mapper.Uid; + +import java.io.IOException; +import java.util.function.IntConsumer; +import java.util.function.Predicate; + +/** + * A query that selects all docs that do NOT belong in the current shards this query is executed on. + * It can be used to split a shard into N shards marking every document that doesn't belong into the shard + * as deleted. See {@link org.apache.lucene.index.IndexWriter#deleteDocuments(Query...)} + */ +final class ShardSplittingQuery extends Query { + private final IndexMetaData indexMetaData; + private final int shardId; + + ShardSplittingQuery(IndexMetaData indexMetaData, int shardId) { + if (indexMetaData.getCreationVersion().before(Version.V_6_0_0_rc2)) { + throw new IllegalArgumentException("Splitting query can only be executed on an index created with version " + + Version.V_6_0_0_rc2 + " or higher"); + } + this.indexMetaData = indexMetaData; + this.shardId = shardId; + } + + @Override + public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) { + return new ConstantScoreWeight(this, boost) { + @Override + public String toString() { + return "weight(delete docs query)"; + } + + @Override + public Scorer scorer(LeafReaderContext context) throws IOException { + LeafReader leafReader = context.reader(); + FixedBitSet bitSet = new FixedBitSet(leafReader.maxDoc()); + Terms terms = leafReader.terms(RoutingFieldMapper.NAME); + Predicate includeInShard = ref -> { + int targetShardId = OperationRouting.generateShardId(indexMetaData, + Uid.decodeId(ref.bytes, ref.offset, ref.length), null); + return shardId == targetShardId; + }; + if (terms == null) { // this is the common case - no partitioning and no _routing values + assert indexMetaData.isRoutingPartitionedIndex() == false; + findSplitDocs(IdFieldMapper.NAME, includeInShard, leafReader, bitSet::set); + } else { + if (indexMetaData.isRoutingPartitionedIndex()) { + // this is the heaviest invariant. Here we have to visit all docs stored fields do extract _id and _routing + // this this index is routing partitioned. + Visitor visitor = new Visitor(); + return new ConstantScoreScorer(this, score(), + new RoutingPartitionedDocIdSetIterator(leafReader, visitor)); + } else { + // in the _routing case we first go and find all docs that have a routing value and mark the ones we have to delete + findSplitDocs(RoutingFieldMapper.NAME, ref -> { + int targetShardId = OperationRouting.generateShardId(indexMetaData, null, ref.utf8ToString()); + return shardId == targetShardId; + }, leafReader, bitSet::set); + // now if we have a mixed index where some docs have a _routing value and some don't we have to exclude the ones + // with a routing value from the next iteration an delete / select based on the ID. + if (terms.getDocCount() != leafReader.maxDoc()) { + // this is a special case where some of the docs have no routing values this sucks but it's possible today + FixedBitSet hasRoutingValue = new FixedBitSet(leafReader.maxDoc()); + findSplitDocs(RoutingFieldMapper.NAME, ref -> false, leafReader, + hasRoutingValue::set); + findSplitDocs(IdFieldMapper.NAME, includeInShard, leafReader, docId -> { + if (hasRoutingValue.get(docId) == false) { + bitSet.set(docId); + } + }); + } + } + } + return new ConstantScoreScorer(this, score(), new BitSetIterator(bitSet, bitSet.length())); + } + + + }; + } + + @Override + public String toString(String field) { + return "shard_splitting_query"; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + ShardSplittingQuery that = (ShardSplittingQuery) o; + + if (shardId != that.shardId) return false; + return indexMetaData.equals(that.indexMetaData); + } + + @Override + public int hashCode() { + int result = indexMetaData.hashCode(); + result = 31 * result + shardId; + return classHash() ^ result; + } + + private static void findSplitDocs(String idField, Predicate includeInShard, + LeafReader leafReader, IntConsumer consumer) throws IOException { + Terms terms = leafReader.terms(idField); + TermsEnum iterator = terms.iterator(); + BytesRef idTerm; + PostingsEnum postingsEnum = null; + while ((idTerm = iterator.next()) != null) { + if (includeInShard.test(idTerm) == false) { + postingsEnum = iterator.postings(postingsEnum); + int doc; + while ((doc = postingsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + consumer.accept(doc); + } + } + } + } + + private static final class Visitor extends StoredFieldVisitor { + int leftToVisit = 2; + final BytesRef spare = new BytesRef(); + String routing; + String id; + + void reset() { + routing = id = null; + leftToVisit = 2; + } + + @Override + public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException { + switch (fieldInfo.name) { + case IdFieldMapper.NAME: + id = Uid.decodeId(value); + break; + default: + throw new IllegalStateException("Unexpected field: " + fieldInfo.name); + } + } + + @Override + public void stringField(FieldInfo fieldInfo, byte[] value) throws IOException { + spare.bytes = value; + spare.offset = 0; + spare.length = value.length; + switch (fieldInfo.name) { + case RoutingFieldMapper.NAME: + routing = spare.utf8ToString(); + break; + default: + throw new IllegalStateException("Unexpected field: " + fieldInfo.name); + } + } + + @Override + public Status needsField(FieldInfo fieldInfo) throws IOException { + // we don't support 5.x so no need for the uid field + switch (fieldInfo.name) { + case IdFieldMapper.NAME: + case RoutingFieldMapper.NAME: + leftToVisit--; + return Status.YES; + default: + return leftToVisit == 0 ? Status.STOP : Status.NO; + } + } + } + + /** + * This two phase iterator visits every live doc and selects all docs that don't belong into this + * shard based on their id and routing value. This is only used in a routing partitioned index. + */ + private final class RoutingPartitionedDocIdSetIterator extends TwoPhaseIterator { + private final LeafReader leafReader; + private final Visitor visitor; + + RoutingPartitionedDocIdSetIterator(LeafReader leafReader, Visitor visitor) { + super(DocIdSetIterator.all(leafReader.maxDoc())); // we iterate all live-docs + this.leafReader = leafReader; + this.visitor = visitor; + } + + @Override + public boolean matches() throws IOException { + int doc = approximation.docID(); + visitor.reset(); + leafReader.document(doc, visitor); + int targetShardId = OperationRouting.generateShardId(indexMetaData, visitor.id, visitor.routing); + return targetShardId != shardId; + } + + @Override + public float matchCost() { + return 42; // that's obvious, right? + } + } +} + + diff --git a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java index e5053fc7882e0..b59ab14961769 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java +++ b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java @@ -31,6 +31,7 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.cluster.routing.RecoverySource; @@ -107,13 +108,16 @@ boolean recoverFromLocalShards(BiConsumer mappingUpdate if (indices.size() > 1) { throw new IllegalArgumentException("can't add shards from more than one index"); } - IndexMetaData indexMetaData = shards.get(0).getIndexMetaData(); - for (ObjectObjectCursor mapping : indexMetaData.getMappings()) { + IndexMetaData sourceMetaData = shards.get(0).getIndexMetaData(); + for (ObjectObjectCursor mapping : sourceMetaData.getMappings()) { mappingUpdateConsumer.accept(mapping.key, mapping.value); } - indexShard.mapperService().merge(indexMetaData, MapperService.MergeReason.MAPPING_RECOVERY, true); + indexShard.mapperService().merge(sourceMetaData, MapperService.MergeReason.MAPPING_RECOVERY, true); // now that the mapping is merged we can validate the index sort configuration. Sort indexSort = indexShard.getIndexSort(); + final boolean isSplit = sourceMetaData.getNumberOfShards() < indexShard.indexSettings().getNumberOfShards(); + assert isSplit == false || sourceMetaData.getCreationVersion().onOrAfter(Version.V_6_0_0_alpha1) : "for split we require a " + + "single type but the index is created before 6.0.0"; return executeRecovery(indexShard, () -> { logger.debug("starting recovery from local shards {}", shards); try { @@ -122,7 +126,8 @@ boolean recoverFromLocalShards(BiConsumer mappingUpdate final long maxSeqNo = shards.stream().mapToLong(LocalShardSnapshot::maxSeqNo).max().getAsLong(); final long maxUnsafeAutoIdTimestamp = shards.stream().mapToLong(LocalShardSnapshot::maxUnsafeAutoIdTimestamp).max().getAsLong(); - addIndices(indexShard.recoveryState().getIndex(), directory, indexSort, sources, maxSeqNo, maxUnsafeAutoIdTimestamp); + addIndices(indexShard.recoveryState().getIndex(), directory, indexSort, sources, maxSeqNo, maxUnsafeAutoIdTimestamp, + indexShard.indexSettings().getIndexMetaData(), indexShard.shardId().id(), isSplit); internalRecoverFromStore(indexShard); // just trigger a merge to do housekeeping on the // copied segments - we will also see them in stats etc. @@ -136,13 +141,9 @@ boolean recoverFromLocalShards(BiConsumer mappingUpdate return false; } - void addIndices( - final RecoveryState.Index indexRecoveryStats, - final Directory target, - final Sort indexSort, - final Directory[] sources, - final long maxSeqNo, - final long maxUnsafeAutoIdTimestamp) throws IOException { + void addIndices(final RecoveryState.Index indexRecoveryStats, final Directory target, final Sort indexSort, final Directory[] sources, + final long maxSeqNo, final long maxUnsafeAutoIdTimestamp, IndexMetaData indexMetaData, int shardId, boolean split) + throws IOException { final Directory hardLinkOrCopyTarget = new org.apache.lucene.store.HardlinkCopyDirectoryWrapper(target); IndexWriterConfig iwc = new IndexWriterConfig(null) .setCommitOnClose(false) @@ -154,8 +155,13 @@ void addIndices( if (indexSort != null) { iwc.setIndexSort(indexSort); } + try (IndexWriter writer = new IndexWriter(new StatsDirectoryWrapper(hardLinkOrCopyTarget, indexRecoveryStats), iwc)) { writer.addIndexes(sources); + + if (split) { + writer.deleteDocuments(new ShardSplittingQuery(indexMetaData, shardId)); + } /* * We set the maximum sequence number and the local checkpoint on the target to the maximum of the maximum sequence numbers on * the source shards. This ensures that history after this maximum sequence number can advance and we have correct @@ -272,7 +278,7 @@ private boolean canRecover(IndexShard indexShard) { // got closed on us, just ignore this recovery return false; } - if (!indexShard.routingEntry().primary()) { + if (indexShard.routingEntry().primary() == false) { throw new IndexShardRecoveryException(shardId, "Trying to recover when the shard is in backup state", null); } return true; diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestShrinkIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestShrinkIndexAction.java index 10b46be6760bb..a0071d70758af 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestShrinkIndexAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestShrinkIndexAction.java @@ -19,8 +19,9 @@ package org.elasticsearch.rest.action.admin.indices; -import org.elasticsearch.action.admin.indices.shrink.ShrinkRequest; -import org.elasticsearch.action.admin.indices.shrink.ShrinkResponse; +import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; +import org.elasticsearch.action.admin.indices.shrink.ResizeResponse; +import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.settings.Settings; @@ -52,14 +53,15 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC if (request.param("index") == null) { throw new IllegalArgumentException("no source index"); } - ShrinkRequest shrinkIndexRequest = new ShrinkRequest(request.param("target"), request.param("index")); - request.applyContentParser(parser -> ShrinkRequest.PARSER.parse(parser, shrinkIndexRequest, null)); + ResizeRequest shrinkIndexRequest = new ResizeRequest(request.param("target"), request.param("index")); + shrinkIndexRequest.setResizeType(ResizeType.SHRINK); + request.applyContentParser(parser -> ResizeRequest.PARSER.parse(parser, shrinkIndexRequest, null)); shrinkIndexRequest.timeout(request.paramAsTime("timeout", shrinkIndexRequest.timeout())); shrinkIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", shrinkIndexRequest.masterNodeTimeout())); shrinkIndexRequest.setWaitForActiveShards(ActiveShardCount.parseString(request.param("wait_for_active_shards"))); - return channel -> client.admin().indices().shrinkIndex(shrinkIndexRequest, new AcknowledgedRestListener(channel) { + return channel -> client.admin().indices().resizeIndex(shrinkIndexRequest, new AcknowledgedRestListener(channel) { @Override - public void addCustomFields(XContentBuilder builder, ShrinkResponse response) throws IOException { + public void addCustomFields(XContentBuilder builder, ResizeResponse response) throws IOException { response.addCustomFields(builder); } }); diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSplitIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSplitIndexAction.java new file mode 100644 index 0000000000000..dcc811bd0177b --- /dev/null +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSplitIndexAction.java @@ -0,0 +1,69 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.rest.action.admin.indices; + +import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; +import org.elasticsearch.action.admin.indices.shrink.ResizeResponse; +import org.elasticsearch.action.admin.indices.shrink.ResizeType; +import org.elasticsearch.action.support.ActiveShardCount; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.AcknowledgedRestListener; + +import java.io.IOException; + +public class RestSplitIndexAction extends BaseRestHandler { + public RestSplitIndexAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.PUT, "/{index}/_split/{target}", this); + controller.registerHandler(RestRequest.Method.POST, "/{index}/_split/{target}", this); + } + + @Override + public String getName() { + return "split_index_action"; + } + + @Override + public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + if (request.param("target") == null) { + throw new IllegalArgumentException("no target index"); + } + if (request.param("index") == null) { + throw new IllegalArgumentException("no source index"); + } + ResizeRequest shrinkIndexRequest = new ResizeRequest(request.param("target"), request.param("index")); + shrinkIndexRequest.setResizeType(ResizeType.SPLIT); + request.applyContentParser(parser -> ResizeRequest.PARSER.parse(parser, shrinkIndexRequest, null)); + shrinkIndexRequest.timeout(request.paramAsTime("timeout", shrinkIndexRequest.timeout())); + shrinkIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", shrinkIndexRequest.masterNodeTimeout())); + shrinkIndexRequest.setWaitForActiveShards(ActiveShardCount.parseString(request.param("wait_for_active_shards"))); + return channel -> client.admin().indices().resizeIndex(shrinkIndexRequest, new AcknowledgedRestListener(channel) { + @Override + public void addCustomFields(XContentBuilder builder, ResizeResponse response) throws IOException { + response.addCustomFields(builder); + } + }); + } +} diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java index 3c2e10d181b58..982b9456b8cdf 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java @@ -105,7 +105,7 @@ public void testCreateShrinkIndexToN() { .put("index.blocks.write", true)).get(); ensureGreen(); // now merge source into a 4 shard index - assertAcked(client().admin().indices().prepareShrinkIndex("source", "first_shrink") + assertAcked(client().admin().indices().prepareResizeIndex("source", "first_shrink") .setSettings(Settings.builder() .put("index.number_of_replicas", 0) .put("index.number_of_shards", shardSplits[1]).build()).get()); @@ -127,7 +127,7 @@ public void testCreateShrinkIndexToN() { .put("index.blocks.write", true)).get(); ensureGreen(); // now merge source into a 2 shard index - assertAcked(client().admin().indices().prepareShrinkIndex("first_shrink", "second_shrink") + assertAcked(client().admin().indices().prepareResizeIndex("first_shrink", "second_shrink") .setSettings(Settings.builder() .put("index.number_of_replicas", 0) .put("index.number_of_shards", shardSplits[2]).build()).get()); @@ -211,7 +211,7 @@ public void testShrinkIndexPrimaryTerm() throws Exception { // now merge source into target final Settings shrinkSettings = Settings.builder().put("index.number_of_replicas", 0).put("index.number_of_shards", numberOfTargetShards).build(); - assertAcked(client().admin().indices().prepareShrinkIndex("source", "target").setSettings(shrinkSettings).get()); + assertAcked(client().admin().indices().prepareResizeIndex("source", "target").setSettings(shrinkSettings).get()); ensureGreen(); @@ -264,7 +264,7 @@ public void testCreateShrinkIndex() { // now merge source into a single shard index final boolean createWithReplicas = randomBoolean(); - assertAcked(client().admin().indices().prepareShrinkIndex("source", "target") + assertAcked(client().admin().indices().prepareResizeIndex("source", "target") .setSettings(Settings.builder().put("index.number_of_replicas", createWithReplicas ? 1 : 0).build()).get()); ensureGreen(); @@ -350,7 +350,7 @@ public void testCreateShrinkIndexFails() throws Exception { ensureGreen(); // now merge source into a single shard index - client().admin().indices().prepareShrinkIndex("source", "target") + client().admin().indices().prepareResizeIndex("source", "target") .setWaitForActiveShards(ActiveShardCount.NONE) .setSettings(Settings.builder() .put("index.routing.allocation.exclude._name", mergeNode) // we manually exclude the merge node to forcefully fuck it up @@ -436,16 +436,16 @@ public void testCreateShrinkWithIndexSort() throws Exception { // check that index sort cannot be set on the target index IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, - () -> client().admin().indices().prepareShrinkIndex("source", "target") + () -> client().admin().indices().prepareResizeIndex("source", "target") .setSettings(Settings.builder() .put("index.number_of_replicas", 0) .put("index.number_of_shards", "2") .put("index.sort.field", "foo") .build()).get()); - assertThat(exc.getMessage(), containsString("can't override index sort when shrinking index")); + assertThat(exc.getMessage(), containsString("can't override index sort when resizing an index")); // check that the index sort order of `source` is correctly applied to the `target` - assertAcked(client().admin().indices().prepareShrinkIndex("source", "target") + assertAcked(client().admin().indices().prepareResizeIndex("source", "target") .setSettings(Settings.builder() .put("index.number_of_replicas", 0) .put("index.number_of_shards", "2").build()).get()); diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java new file mode 100644 index 0000000000000..8f24edf8577e4 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java @@ -0,0 +1,462 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.create; + +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.SortedSetSelector; +import org.apache.lucene.search.SortedSetSortField; +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.elasticsearch.action.admin.indices.shrink.ResizeType; +import org.elasticsearch.action.admin.indices.stats.CommonStats; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.action.admin.indices.stats.ShardStats; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.routing.Murmur3HashFunction; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.engine.SegmentsStats; +import org.elasticsearch.index.query.TermsQueryBuilder; +import org.elasticsearch.index.seqno.SeqNoStats; +import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.InternalSettingsPlugin; +import org.elasticsearch.test.VersionUtils; + +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.IntStream; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; + + +public class SplitIndexIT extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return Arrays.asList(InternalSettingsPlugin.class); + } + + public void testCreateSplitIndexToN() { + int[][] possibleShardSplits = new int[][] {{2,4,8}, {3, 6, 12}, {1, 2, 4}}; + int[] shardSplits = randomFrom(possibleShardSplits); + assertEquals(shardSplits[0], (shardSplits[0] * shardSplits[1]) / shardSplits[1]); + assertEquals(shardSplits[1], (shardSplits[1] * shardSplits[2]) / shardSplits[2]); + internalCluster().ensureAtLeastNumDataNodes(2); + final boolean useRouting = randomBoolean(); + final boolean useMixedRouting = useRouting ? randomBoolean() : false; + CreateIndexRequestBuilder createInitialIndex = prepareCreate("source"); + Settings.Builder settings = Settings.builder().put(indexSettings()) + .put("number_of_shards", shardSplits[0]) + .put("index.number_of_routing_shards", shardSplits[2] * randomIntBetween(1, 10)); + if (useRouting && useMixedRouting == false && randomBoolean()) { + settings.put("index.routing_partition_size", randomIntBetween(1, 10)); + createInitialIndex.addMapping("t1", "_routing", "required=true"); + } + logger.info("use routing {} use mixed routing {}", useRouting, useMixedRouting); + createInitialIndex.setSettings(settings).get(); + + int numDocs = randomIntBetween(10, 50); + String[] routingValue = new String[numDocs]; + for (int i = 0; i < numDocs; i++) { + IndexRequestBuilder builder = client().prepareIndex("source", "t1", Integer.toString(i)) + .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON); + if (useRouting) { + String routing = randomRealisticUnicodeOfCodepointLengthBetween(1, 10); + if (useMixedRouting && randomBoolean()) { + routingValue[i] = null; + } else { + routingValue[i] = routing; + } + builder.setRouting(routingValue[i]); + } + builder.get(); + } + + if (randomBoolean()) { + for (int i = 0; i < numDocs; i++) { // let's introduce some updates / deletes on the index + if (randomBoolean()) { + IndexRequestBuilder builder = client().prepareIndex("source", "t1", Integer.toString(i)) + .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON); + if (useRouting) { + builder.setRouting(routingValue[i]); + } + builder.get(); + } + } + } + + ImmutableOpenMap dataNodes = client().admin().cluster().prepareState().get().getState().nodes() + .getDataNodes(); + assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2); + ensureYellow(); + client().admin().indices().prepareUpdateSettings("source") + .setSettings(Settings.builder() + .put("index.blocks.write", true)).get(); + ensureGreen(); + assertAcked(client().admin().indices().prepareResizeIndex("source", "first_split") + .setResizeType(ResizeType.SPLIT) + .setSettings(Settings.builder() + .put("index.number_of_replicas", 0) + .put("index.number_of_shards", shardSplits[1]).build()).get()); + ensureGreen(); + assertHitCount(client().prepareSearch("first_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs); + + for (int i = 0; i < numDocs; i++) { // now update + IndexRequestBuilder builder = client().prepareIndex("first_split", "t1", Integer.toString(i)) + .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON); + if (useRouting) { + builder.setRouting(routingValue[i]); + } + builder.get(); + } + flushAndRefresh(); + assertHitCount(client().prepareSearch("first_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs); + assertHitCount(client().prepareSearch("source").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs); + for (int i = 0; i < numDocs; i++) { + GetResponse getResponse = client().prepareGet("first_split", "t1", Integer.toString(i)).setRouting(routingValue[i]).get(); + assertTrue(getResponse.isExists()); + } + + client().admin().indices().prepareUpdateSettings("first_split") + .setSettings(Settings.builder() + .put("index.blocks.write", true)).get(); + ensureGreen(); + // now split source into a new index + assertAcked(client().admin().indices().prepareResizeIndex("first_split", "second_split") + .setResizeType(ResizeType.SPLIT) + .setSettings(Settings.builder() + .put("index.number_of_replicas", 0) + .put("index.number_of_shards", shardSplits[2]).build()).get()); + ensureGreen(); + assertHitCount(client().prepareSearch("second_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs); + // let it be allocated anywhere and bump replicas + client().admin().indices().prepareUpdateSettings("second_split") + .setSettings(Settings.builder() + .put("index.number_of_replicas", 1)).get(); + ensureGreen(); + assertHitCount(client().prepareSearch("second_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs); + + for (int i = 0; i < numDocs; i++) { // now update + IndexRequestBuilder builder = client().prepareIndex("second_split", "t1", Integer.toString(i)) + .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON); + if (useRouting) { + builder.setRouting(routingValue[i]); + } + builder.get(); + } + flushAndRefresh(); + for (int i = 0; i < numDocs; i++) { + GetResponse getResponse = client().prepareGet("second_split", "t1", Integer.toString(i)).setRouting(routingValue[i]).get(); + assertTrue(getResponse.isExists()); + } + assertHitCount(client().prepareSearch("second_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs); + assertHitCount(client().prepareSearch("first_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs); + assertHitCount(client().prepareSearch("source").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs); + + assertAllUniqueDocs(client().prepareSearch("second_split").setSize(100) + .setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs); + assertAllUniqueDocs(client().prepareSearch("first_split").setSize(100) + .setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs); + assertAllUniqueDocs(client().prepareSearch("source").setSize(100) + .setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs); + + } + + public void assertAllUniqueDocs(SearchResponse response, int numDocs) { + Set ids = new HashSet<>(); + for (int i = 0; i < response.getHits().getHits().length; i++) { + String id = response.getHits().getHits()[i].getId(); + assertTrue("found ID "+ id + " more than once", ids.add(id)); + } + assertEquals(numDocs, ids.size()); + } + + public void testSplitIndexPrimaryTerm() throws Exception { + final List factors = Arrays.asList(1, 2, 4, 8); + final List numberOfShardsFactors = randomSubsetOf(scaledRandomIntBetween(1, factors.size()), factors); + final int numberOfShards = randomSubsetOf(numberOfShardsFactors).stream().reduce(1, (x, y) -> x * y); + final int numberOfTargetShards = numberOfShardsFactors.stream().reduce(2, (x, y) -> x * y); + internalCluster().ensureAtLeastNumDataNodes(2); + prepareCreate("source").setSettings(Settings.builder().put(indexSettings()) + .put("number_of_shards", numberOfShards) + .put("index.number_of_routing_shards", numberOfTargetShards)).get(); + + final ImmutableOpenMap dataNodes = + client().admin().cluster().prepareState().get().getState().nodes().getDataNodes(); + assertThat(dataNodes.size(), greaterThanOrEqualTo(2)); + ensureYellow(); + + // fail random primary shards to force primary terms to increase + final Index source = resolveIndex("source"); + final int iterations = scaledRandomIntBetween(0, 16); + for (int i = 0; i < iterations; i++) { + final String node = randomSubsetOf(1, internalCluster().nodesInclude("source")).get(0); + final IndicesService indexServices = internalCluster().getInstance(IndicesService.class, node); + final IndexService indexShards = indexServices.indexServiceSafe(source); + for (final Integer shardId : indexShards.shardIds()) { + final IndexShard shard = indexShards.getShard(shardId); + if (shard.routingEntry().primary() && randomBoolean()) { + disableAllocation("source"); + shard.failShard("test", new Exception("test")); + // this can not succeed until the shard is failed and a replica is promoted + int id = 0; + while (true) { + // find an ID that routes to the right shard, we will only index to the shard that saw a primary failure + final String s = Integer.toString(id); + final int hash = Math.floorMod(Murmur3HashFunction.hash(s), numberOfShards); + if (hash == shardId) { + final IndexRequest request = + new IndexRequest("source", "type", s).source("{ \"f\": \"" + s + "\"}", XContentType.JSON); + client().index(request).get(); + break; + } else { + id++; + } + } + enableAllocation("source"); + ensureGreen(); + } + } + } + + final Settings.Builder prepareSplitSettings = Settings.builder().put("index.blocks.write", true); + client().admin().indices().prepareUpdateSettings("source").setSettings(prepareSplitSettings).get(); + ensureYellow(); + + final IndexMetaData indexMetaData = indexMetaData(client(), "source"); + final long beforeSplitPrimaryTerm = IntStream.range(0, numberOfShards).mapToLong(indexMetaData::primaryTerm).max().getAsLong(); + + // now split source into target + final Settings splitSettings = + Settings.builder().put("index.number_of_replicas", 0).put("index.number_of_shards", numberOfTargetShards).build(); + assertAcked(client().admin().indices().prepareResizeIndex("source", "target") + .setResizeType(ResizeType.SPLIT) + .setSettings(splitSettings).get()); + + ensureGreen(); + + final IndexMetaData aftersplitIndexMetaData = indexMetaData(client(), "target"); + for (int shardId = 0; shardId < numberOfTargetShards; shardId++) { + assertThat(aftersplitIndexMetaData.primaryTerm(shardId), equalTo(beforeSplitPrimaryTerm + 1)); + } + } + + private static IndexMetaData indexMetaData(final Client client, final String index) { + final ClusterStateResponse clusterStateResponse = client.admin().cluster().state(new ClusterStateRequest()).actionGet(); + return clusterStateResponse.getState().metaData().index(index); + } + + public void testCreateSplitIndex() { + internalCluster().ensureAtLeastNumDataNodes(2); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0_rc2, Version.CURRENT); + prepareCreate("source").setSettings(Settings.builder().put(indexSettings()) + .put("number_of_shards", 1) + .put("index.version.created", version) + .put("index.number_of_routing_shards", 2) + ).get(); + final int docs = randomIntBetween(0, 128); + for (int i = 0; i < docs; i++) { + client().prepareIndex("source", "type") + .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get(); + } + ImmutableOpenMap dataNodes = + client().admin().cluster().prepareState().get().getState().nodes().getDataNodes(); + assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2); + // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node + // if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due + // to the require._name below. + ensureGreen(); + // relocate all shards to one node such that we can merge it. + client().admin().indices().prepareUpdateSettings("source") + .setSettings(Settings.builder() + .put("index.blocks.write", true)).get(); + ensureGreen(); + + final IndicesStatsResponse sourceStats = client().admin().indices().prepareStats("source").setSegments(true).get(); + + // disable rebalancing to be able to capture the right stats. balancing can move the target primary + // making it hard to pin point the source shards. + client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put( + EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), "none" + )).get(); + try { + + final boolean createWithReplicas = randomBoolean(); + assertAcked(client().admin().indices().prepareResizeIndex("source", "target") + .setResizeType(ResizeType.SPLIT) + .setSettings(Settings.builder() + .put("index.number_of_replicas", createWithReplicas ? 1 : 0) + .put("index.number_of_shards", 2).build()).get()); + ensureGreen(); + + final ClusterState state = client().admin().cluster().prepareState().get().getState(); + DiscoveryNode mergeNode = state.nodes().get(state.getRoutingTable().index("target").shard(0).primaryShard().currentNodeId()); + logger.info("split node {}", mergeNode); + + final long maxSeqNo = Arrays.stream(sourceStats.getShards()) + .filter(shard -> shard.getShardRouting().currentNodeId().equals(mergeNode.getId())) + .map(ShardStats::getSeqNoStats).mapToLong(SeqNoStats::getMaxSeqNo).max().getAsLong(); + final long maxUnsafeAutoIdTimestamp = Arrays.stream(sourceStats.getShards()) + .filter(shard -> shard.getShardRouting().currentNodeId().equals(mergeNode.getId())) + .map(ShardStats::getStats) + .map(CommonStats::getSegments) + .mapToLong(SegmentsStats::getMaxUnsafeAutoIdTimestamp) + .max() + .getAsLong(); + + final IndicesStatsResponse targetStats = client().admin().indices().prepareStats("target").get(); + for (final ShardStats shardStats : targetStats.getShards()) { + final SeqNoStats seqNoStats = shardStats.getSeqNoStats(); + final ShardRouting shardRouting = shardStats.getShardRouting(); + assertThat("failed on " + shardRouting, seqNoStats.getMaxSeqNo(), equalTo(maxSeqNo)); + assertThat("failed on " + shardRouting, seqNoStats.getLocalCheckpoint(), equalTo(maxSeqNo)); + assertThat("failed on " + shardRouting, + shardStats.getStats().getSegments().getMaxUnsafeAutoIdTimestamp(), equalTo(maxUnsafeAutoIdTimestamp)); + } + + final int size = docs > 0 ? 2 * docs : 1; + assertHitCount(client().prepareSearch("target").setSize(size).setQuery(new TermsQueryBuilder("foo", "bar")).get(), docs); + + if (createWithReplicas == false) { + // bump replicas + client().admin().indices().prepareUpdateSettings("target") + .setSettings(Settings.builder() + .put("index.number_of_replicas", 1)).get(); + ensureGreen(); + assertHitCount(client().prepareSearch("target").setSize(size).setQuery(new TermsQueryBuilder("foo", "bar")).get(), docs); + } + + for (int i = docs; i < 2 * docs; i++) { + client().prepareIndex("target", "type") + .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get(); + } + flushAndRefresh(); + assertHitCount(client().prepareSearch("target").setSize(2 * size).setQuery(new TermsQueryBuilder("foo", "bar")).get(), + 2 * docs); + assertHitCount(client().prepareSearch("source").setSize(size).setQuery(new TermsQueryBuilder("foo", "bar")).get(), docs); + GetSettingsResponse target = client().admin().indices().prepareGetSettings("target").get(); + assertEquals(version, target.getIndexToSettings().get("target").getAsVersion("index.version.created", null)); + } finally { + // clean up + client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put( + EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), (String)null + )).get(); + } + + } + + public void testCreateSplitWithIndexSort() throws Exception { + SortField expectedSortField = new SortedSetSortField("id", true, SortedSetSelector.Type.MAX); + expectedSortField.setMissingValue(SortedSetSortField.STRING_FIRST); + Sort expectedIndexSort = new Sort(expectedSortField); + internalCluster().ensureAtLeastNumDataNodes(2); + prepareCreate("source") + .setSettings( + Settings.builder() + .put(indexSettings()) + .put("sort.field", "id") + .put("index.number_of_routing_shards", 16) + .put("sort.order", "desc") + .put("number_of_shards", 2) + .put("number_of_replicas", 0) + ) + .addMapping("type", "id", "type=keyword,doc_values=true") + .get(); + for (int i = 0; i < 20; i++) { + client().prepareIndex("source", "type", Integer.toString(i)) + .setSource("{\"foo\" : \"bar\", \"id\" : " + i + "}", XContentType.JSON).get(); + } + ImmutableOpenMap dataNodes = client().admin().cluster().prepareState().get().getState().nodes() + .getDataNodes(); + assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2); + DiscoveryNode[] discoveryNodes = dataNodes.values().toArray(DiscoveryNode.class); + String mergeNode = discoveryNodes[0].getName(); + // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node + // if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due + // to the require._name below. + ensureGreen(); + + flushAndRefresh(); + assertSortedSegments("source", expectedIndexSort); + + client().admin().indices().prepareUpdateSettings("source") + .setSettings(Settings.builder() + .put("index.blocks.write", true)).get(); + ensureYellow(); + + // check that index sort cannot be set on the target index + IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, + () -> client().admin().indices().prepareResizeIndex("source", "target") + .setResizeType(ResizeType.SPLIT) + .setSettings(Settings.builder() + .put("index.number_of_replicas", 0) + .put("index.number_of_shards", 4) + .put("index.sort.field", "foo") + .build()).get()); + assertThat(exc.getMessage(), containsString("can't override index sort when resizing an index")); + + // check that the index sort order of `source` is correctly applied to the `target` + assertAcked(client().admin().indices().prepareResizeIndex("source", "target") + .setResizeType(ResizeType.SPLIT) + .setSettings(Settings.builder() + .put("index.number_of_replicas", 0) + .put("index.number_of_shards", 4).build()).get()); + ensureGreen(); + flushAndRefresh(); + GetSettingsResponse settingsResponse = + client().admin().indices().prepareGetSettings("target").execute().actionGet(); + assertEquals(settingsResponse.getSetting("target", "index.sort.field"), "id"); + assertEquals(settingsResponse.getSetting("target", "index.sort.order"), "desc"); + assertSortedSegments("target", expectedIndexSort); + + // ... and that the index sort is also applied to updates + for (int i = 20; i < 40; i++) { + client().prepareIndex("target", "type") + .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get(); + } + flushAndRefresh(); + assertSortedSegments("target", expectedIndexSort); + } +} diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java similarity index 87% rename from core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java rename to core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java index 83e9cf89d9c75..b03b043f03e14 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java @@ -28,7 +28,6 @@ import org.elasticsearch.cluster.EmptyClusterInfoService; import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; @@ -49,7 +48,7 @@ import static java.util.Collections.emptyMap; -public class TransportShrinkActionTests extends ESTestCase { +public class TransportResizeActionTests extends ESTestCase { private ClusterState createClusterState(String name, int numShards, int numReplicas, Settings settings) { MetaData.Builder metaBuilder = MetaData.builder(); @@ -72,20 +71,20 @@ public void testErrorCondition() { Settings.builder().put("index.blocks.write", true).build()); assertTrue( expectThrows(IllegalStateException.class, () -> - TransportShrinkAction.prepareCreateIndexRequest(new ShrinkRequest("target", "source"), state, - (i) -> new DocsStats(Integer.MAX_VALUE, between(1, 1000), between(1, 100)), new IndexNameExpressionResolver(Settings.EMPTY)) + TransportResizeAction.prepareCreateIndexRequest(new ResizeRequest("target", "source"), state, + (i) -> new DocsStats(Integer.MAX_VALUE, between(1, 1000), between(1, 100)), "source", "target") ).getMessage().startsWith("Can't merge index with more than [2147483519] docs - too many documents in shards ")); assertTrue( expectThrows(IllegalStateException.class, () -> { - ShrinkRequest req = new ShrinkRequest("target", "source"); - req.getShrinkIndexRequest().settings(Settings.builder().put("index.number_of_shards", 4)); + ResizeRequest req = new ResizeRequest("target", "source"); + req.getTargetIndexRequest().settings(Settings.builder().put("index.number_of_shards", 4)); ClusterState clusterState = createClusterState("source", 8, 1, Settings.builder().put("index.blocks.write", true).build()); - TransportShrinkAction.prepareCreateIndexRequest(req, clusterState, - (i) -> i == 2 || i == 3 ? new DocsStats(Integer.MAX_VALUE / 2, between(1, 1000), between(1, 10000)) : null, - new IndexNameExpressionResolver(Settings.EMPTY)); + TransportResizeAction.prepareCreateIndexRequest(req, clusterState, + (i) -> i == 2 || i == 3 ? new DocsStats(Integer.MAX_VALUE / 2, between(1, 1000), between(1, 10000)) : null + , "source", "target"); } ).getMessage().startsWith("Can't merge index with more than [2147483519] docs - too many documents in shards ")); @@ -105,8 +104,8 @@ public void testErrorCondition() { routingTable.index("source").shardsWithState(ShardRoutingState.INITIALIZING)).routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); - TransportShrinkAction.prepareCreateIndexRequest(new ShrinkRequest("target", "source"), clusterState, - (i) -> new DocsStats(between(1, 1000), between(1, 1000), between(0, 10000)), new IndexNameExpressionResolver(Settings.EMPTY)); + TransportResizeAction.prepareCreateIndexRequest(new ResizeRequest("target", "source"), clusterState, + (i) -> new DocsStats(between(1, 1000), between(1, 1000), between(0, 10000)), "source", "target"); } public void testShrinkIndexSettings() { @@ -129,14 +128,13 @@ public void testShrinkIndexSettings() { clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); int numSourceShards = clusterState.metaData().index(indexName).getNumberOfShards(); DocsStats stats = new DocsStats(between(0, (IndexWriter.MAX_DOCS) / numSourceShards), between(1, 1000), between(1, 10000)); - ShrinkRequest target = new ShrinkRequest("target", indexName); + ResizeRequest target = new ResizeRequest("target", indexName); final ActiveShardCount activeShardCount = randomBoolean() ? ActiveShardCount.ALL : ActiveShardCount.ONE; target.setWaitForActiveShards(activeShardCount); - CreateIndexClusterStateUpdateRequest request = TransportShrinkAction.prepareCreateIndexRequest( - target, clusterState, (i) -> stats, - new IndexNameExpressionResolver(Settings.EMPTY)); - assertNotNull(request.shrinkFrom()); - assertEquals(indexName, request.shrinkFrom().getName()); + CreateIndexClusterStateUpdateRequest request = TransportResizeAction.prepareCreateIndexRequest( + target, clusterState, (i) -> stats, indexName, "target"); + assertNotNull(request.recoverFrom()); + assertEquals(indexName, request.recoverFrom().getName()); assertEquals("1", request.settings().get("index.number_of_shards")); assertEquals("shrink_index", request.cause()); assertEquals(request.waitForActiveShards(), activeShardCount); diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java b/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java index 81acd138d26fb..6fd3d66c8f81b 100644 --- a/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java @@ -34,6 +34,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.NodeVersionAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.RebalanceOnlyWhenActiveAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.ReplicaAfterPrimaryActiveAllocationDecider; +import org.elasticsearch.cluster.routing.allocation.decider.ResizeAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.SnapshotInProgressAllocationDecider; @@ -174,6 +175,7 @@ public void testShardsAllocatorFactoryNull() { public void testAllocationDeciderOrder() { List> expectedDeciders = Arrays.asList( MaxRetryAllocationDecider.class, + ResizeAllocationDecider.class, ReplicaAfterPrimaryActiveAllocationDecider.class, RebalanceOnlyWhenActiveAllocationDecider.class, ClusterRebalanceAllocationDecider.class, diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java index 4dd757c140311..f44d0b7c4036e 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; +import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlock; @@ -258,8 +259,8 @@ public void testIndexRemovalOnFailure() throws Exception { public void testShrinkIndexIgnoresTemplates() throws Exception { final Index source = new Index("source_idx", "aaa111bbb222"); - when(request.shrinkFrom()).thenReturn(source); - + when(request.recoverFrom()).thenReturn(source); + when(request.resizeType()).thenReturn(ResizeType.SHRINK); currentStateMetaDataBuilder.put(createIndexMetaDataBuilder("source_idx", "aaa111bbb222", 2, 2)); routingTableBuilder.add(createIndexRoutingTableWithStartedShards(source)); diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java index fa56c756fcc35..e83d1fa706cfd 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import java.util.Collections; import java.util.Set; import static org.hamcrest.Matchers.is; @@ -84,21 +85,12 @@ public void testIndexMetaDataSerialization() throws IOException { } public void testGetRoutingFactor() { - int numberOfReplicas = randomIntBetween(0, 10); - IndexMetaData metaData = IndexMetaData.builder("foo") - .settings(Settings.builder() - .put("index.version.created", 1) - .put("index.number_of_shards", 32) - .put("index.number_of_replicas", numberOfReplicas) - .build()) - .creationDate(randomLong()) - .build(); Integer numShard = randomFrom(1, 2, 4, 8, 16); - int routingFactor = IndexMetaData.getRoutingFactor(metaData, numShard); - assertEquals(routingFactor * numShard, metaData.getNumberOfShards()); + int routingFactor = IndexMetaData.getRoutingFactor(32, numShard); + assertEquals(routingFactor * numShard, 32); - Integer brokenNumShards = randomFrom(3, 5, 9, 12, 29, 42, 64); - expectThrows(IllegalArgumentException.class, () -> IndexMetaData.getRoutingFactor(metaData, brokenNumShards)); + Integer brokenNumShards = randomFrom(3, 5, 9, 12, 29, 42); + expectThrows(IllegalArgumentException.class, () -> IndexMetaData.getRoutingFactor(32, brokenNumShards)); } public void testSelectShrinkShards() { @@ -125,6 +117,64 @@ public void testSelectShrinkShards() { expectThrows(IllegalArgumentException.class, () -> IndexMetaData.selectShrinkShards(8, metaData, 8)).getMessage()); } + public void testSelectResizeShards() { + IndexMetaData split = IndexMetaData.builder("foo") + .settings(Settings.builder() + .put("index.version.created", 1) + .put("index.number_of_shards", 2) + .put("index.number_of_replicas", 0) + .build()) + .creationDate(randomLong()) + .build(); + + IndexMetaData shrink = IndexMetaData.builder("foo") + .settings(Settings.builder() + .put("index.version.created", 1) + .put("index.number_of_shards", 32) + .put("index.number_of_replicas", 0) + .build()) + .creationDate(randomLong()) + .build(); + int numTargetShards = randomFrom(4, 6, 8, 12); + int shard = randomIntBetween(0, numTargetShards-1); + assertEquals(Collections.singleton(IndexMetaData.selectSplitShard(shard, split, numTargetShards)), + IndexMetaData.selectRecoverFromShards(shard, split, numTargetShards)); + + numTargetShards = randomFrom(1, 2, 4, 8, 16); + shard = randomIntBetween(0, numTargetShards-1); + assertEquals(IndexMetaData.selectShrinkShards(shard, shrink, numTargetShards), + IndexMetaData.selectRecoverFromShards(shard, shrink, numTargetShards)); + + assertEquals("can't select recover from shards if both indices have the same number of shards", + expectThrows(IllegalArgumentException.class, () -> IndexMetaData.selectRecoverFromShards(0, shrink, 32)).getMessage()); + } + + public void testSelectSplitShard() { + IndexMetaData metaData = IndexMetaData.builder("foo") + .settings(Settings.builder() + .put("index.version.created", 1) + .put("index.number_of_shards", 2) + .put("index.number_of_replicas", 0) + .build()) + .creationDate(randomLong()) + .setRoutingNumShards(4) + .build(); + ShardId shardId = IndexMetaData.selectSplitShard(0, metaData, 4); + assertEquals(0, shardId.getId()); + shardId = IndexMetaData.selectSplitShard(1, metaData, 4); + assertEquals(0, shardId.getId()); + shardId = IndexMetaData.selectSplitShard(2, metaData, 4); + assertEquals(1, shardId.getId()); + shardId = IndexMetaData.selectSplitShard(3, metaData, 4); + assertEquals(1, shardId.getId()); + + assertEquals("the number of target shards (0) must be greater than the shard id: 0", + expectThrows(IllegalArgumentException.class, () -> IndexMetaData.selectSplitShard(0, metaData, 0)).getMessage()); + + assertEquals("the number of source shards [2] must be a must be a factor of [3]", + expectThrows(IllegalArgumentException.class, () -> IndexMetaData.selectSplitShard(0, metaData, 3)).getMessage()); + } + public void testIndexFormat() { Settings defaultSettings = Settings.builder() .put("index.version.created", 1) @@ -156,4 +206,26 @@ public void testIndexFormat() { assertThat(metaData.getSettings().getAsInt(IndexMetaData.INDEX_FORMAT_SETTING.getKey(), 0), is(0)); } } + + public void testNumberOfRoutingShards() { + Settings build = Settings.builder().put("index.number_of_shards", 5).put("index.number_of_routing_shards", 10).build(); + assertEquals(10, IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(build).intValue()); + + build = Settings.builder().put("index.number_of_shards", 5).put("index.number_of_routing_shards", 5).build(); + assertEquals(5, IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(build).intValue()); + + int numShards = randomIntBetween(1, 10); + build = Settings.builder().put("index.number_of_shards", numShards).build(); + assertEquals(numShards, IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(build).intValue()); + + Settings lessThanSettings = Settings.builder().put("index.number_of_shards", 8).put("index.number_of_routing_shards", 4).build(); + IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, + () -> IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(lessThanSettings)); + assertEquals("index.number_of_routing_shards [4] must be >= index.number_of_shards [8]", iae.getMessage()); + + Settings notAFactorySettings = Settings.builder().put("index.number_of_shards", 2).put("index.number_of_routing_shards", 3).build(); + iae = expectThrows(IllegalArgumentException.class, + () -> IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(notAFactorySettings)); + assertEquals("the number of source shards [2] must be a must be a factor of [3]", iae.getMessage()); + } } diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index 7bfc7872f816a..39e4a18440931 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.cluster.metadata; import org.elasticsearch.Version; +import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.EmptyClusterInfoService; @@ -33,6 +34,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.ResourceAlreadyExistsException; @@ -43,6 +45,7 @@ import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.HashSet; import java.util.List; @@ -75,6 +78,12 @@ public static boolean isShrinkable(int source, int target) { return target * x == source; } + public static boolean isSplitable(int source, int target) { + int x = target / source; + assert source < target : source + " >= " + target; + return source * x == target; + } + public void testValidateShrinkIndex() { int numShards = randomIntBetween(2, 42); ClusterState state = createClusterState("source", numShards, randomIntBetween(0, 10), @@ -90,29 +99,28 @@ public void testValidateShrinkIndex() { MetaDataCreateIndexService.validateShrinkIndex(state, "no such index", Collections.emptySet(), "target", Settings.EMPTY) ).getMessage()); + Settings targetSettings = Settings.builder().put("index.number_of_shards", 1).build(); assertEquals("can't shrink an index with only one shard", expectThrows(IllegalArgumentException.class, () -> MetaDataCreateIndexService.validateShrinkIndex(createClusterState("source", - 1, 0, Settings.builder().put("index.blocks.write", true).build()), "source", Collections.emptySet(), - "target", Settings.EMPTY) - ).getMessage()); + 1, 0, Settings.builder().put("index.blocks.write", true).build()), "source", + Collections.emptySet(), "target", targetSettings)).getMessage()); - assertEquals("the number of target shards must be less that the number of source shards", + assertEquals("the number of target shards [10] must be less that the number of source shards [5]", expectThrows(IllegalArgumentException.class, () -> MetaDataCreateIndexService.validateShrinkIndex(createClusterState("source", - 5, 0, Settings.builder().put("index.blocks.write", true).build()), "source", Collections.emptySet(), - "target", Settings.builder().put("index.number_of_shards", 10).build()) - ).getMessage()); + 5, 0, Settings.builder().put("index.blocks.write", true).build()), "source", + Collections.emptySet(), "target", Settings.builder().put("index.number_of_shards", 10).build())).getMessage()); - assertEquals("index source must be read-only to shrink index. use \"index.blocks.write=true\"", + assertEquals("index source must be read-only to resize index. use \"index.blocks.write=true\"", expectThrows(IllegalStateException.class, () -> MetaDataCreateIndexService.validateShrinkIndex( createClusterState("source", randomIntBetween(2, 100), randomIntBetween(0, 10), Settings.EMPTY) - , "source", Collections.emptySet(), "target", Settings.EMPTY) + , "source", Collections.emptySet(), "target", targetSettings) ).getMessage()); assertEquals("index source must have all shards allocated on the same node to shrink index", expectThrows(IllegalStateException.class, () -> - MetaDataCreateIndexService.validateShrinkIndex(state, "source", Collections.emptySet(), "target", Settings.EMPTY) + MetaDataCreateIndexService.validateShrinkIndex(state, "source", Collections.emptySet(), "target", targetSettings) ).getMessage()); assertEquals("the number of source shards [8] must be a must be a multiple of [3]", @@ -122,10 +130,10 @@ public void testValidateShrinkIndex() { Settings.builder().put("index.number_of_shards", 3).build()) ).getMessage()); - assertEquals("mappings are not allowed when shrinking indices, all mappings are copied from the source index", + assertEquals("mappings are not allowed when resizing indices, all mappings are copied from the source index", expectThrows(IllegalArgumentException.class, () -> { MetaDataCreateIndexService.validateShrinkIndex(state, "source", Collections.singleton("foo"), - "target", Settings.EMPTY); + "target", targetSettings); } ).getMessage()); @@ -151,11 +159,78 @@ public void testValidateShrinkIndex() { Settings.builder().put("index.number_of_shards", targetShards).build()); } - public void testShrinkIndexSettings() { + public void testValidateSplitIndex() { + int numShards = randomIntBetween(1, 42); + Settings targetSettings = Settings.builder().put("index.number_of_shards", numShards * 2).build(); + ClusterState state = createClusterState("source", numShards, randomIntBetween(0, 10), + Settings.builder().put("index.blocks.write", true).build()); + + assertEquals("index [source] already exists", + expectThrows(ResourceAlreadyExistsException.class, () -> + MetaDataCreateIndexService.validateSplitIndex(state, "target", Collections.emptySet(), "source", targetSettings) + ).getMessage()); + + assertEquals("no such index", + expectThrows(IndexNotFoundException.class, () -> + MetaDataCreateIndexService.validateSplitIndex(state, "no such index", Collections.emptySet(), "target", targetSettings) + ).getMessage()); + + assertEquals("the number of source shards [10] must be less that the number of target shards [5]", + expectThrows(IllegalArgumentException.class, () -> MetaDataCreateIndexService.validateSplitIndex(createClusterState("source", + 10, 0, Settings.builder().put("index.blocks.write", true).build()), "source", Collections.emptySet(), + "target", Settings.builder().put("index.number_of_shards", 5).build()) + ).getMessage()); + + + assertEquals("index source must be read-only to resize index. use \"index.blocks.write=true\"", + expectThrows(IllegalStateException.class, () -> + MetaDataCreateIndexService.validateSplitIndex( + createClusterState("source", randomIntBetween(2, 100), randomIntBetween(0, 10), Settings.EMPTY) + , "source", Collections.emptySet(), "target", targetSettings) + ).getMessage()); + + + assertEquals("the number of source shards [3] must be a must be a factor of [4]", + expectThrows(IllegalArgumentException.class, () -> + MetaDataCreateIndexService.validateSplitIndex(createClusterState("source", 3, randomIntBetween(0, 10), + Settings.builder().put("index.blocks.write", true).build()), "source", Collections.emptySet(), "target", + Settings.builder().put("index.number_of_shards", 4).build()) + ).getMessage()); + + assertEquals("mappings are not allowed when resizing indices, all mappings are copied from the source index", + expectThrows(IllegalArgumentException.class, () -> { + MetaDataCreateIndexService.validateSplitIndex(state, "source", Collections.singleton("foo"), + "target", targetSettings); + } + ).getMessage()); + + + ClusterState clusterState = ClusterState.builder(createClusterState("source", numShards, 0, + Settings.builder().put("index.blocks.write", true).build())).nodes(DiscoveryNodes.builder().add(newNode("node1"))) + .build(); + AllocationService service = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY, + Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))), + new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE); + + RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + // now we start the shard + routingTable = service.applyStartedShards(clusterState, + routingTable.index("source").shardsWithState(ShardRoutingState.INITIALIZING)).routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + int targetShards; + do { + targetShards = randomIntBetween(numShards+1, 100); + } while (isSplitable(numShards, targetShards) == false); + MetaDataCreateIndexService.validateSplitIndex(clusterState, "source", Collections.emptySet(), "target", + Settings.builder().put("index.number_of_shards", targetShards).build()); + } + + public void testResizeIndexSettings() { String indexName = randomAlphaOfLength(10); List versions = Arrays.asList(VersionUtils.randomVersion(random()), VersionUtils.randomVersion(random()), VersionUtils.randomVersion(random())); - versions.sort((l, r) -> Long.compare(l.id, r.id)); + versions.sort(Comparator.comparingLong(l -> l.id)); Version version = versions.get(0); Version minCompat = versions.get(1); Version upgraded = versions.get(2); @@ -182,8 +257,9 @@ public void testShrinkIndexSettings() { clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); Settings.Builder builder = Settings.builder(); - MetaDataCreateIndexService.prepareShrinkIndexSettings( - clusterState, Collections.emptySet(), builder, clusterState.metaData().index(indexName).getIndex(), "target"); + builder.put("index.number_of_shards", 1); + MetaDataCreateIndexService.prepareResizeIndexSettings(clusterState, Collections.emptySet(), builder, + clusterState.metaData().index(indexName).getIndex(), "target", ResizeType.SHRINK); assertEquals("similarity settings must be copied", "BM25", builder.build().get("index.similarity.default.type")); assertEquals("analysis settings must be copied", "keyword", builder.build().get("index.analysis.analyzer.my_analyzer.tokenizer")); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java index 498edee12f90a..1f8de1ca02fd7 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java @@ -23,9 +23,7 @@ import org.elasticsearch.action.support.replication.ClusterStateCreationUtils; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.Nullable; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; @@ -54,6 +52,7 @@ public class OperationRoutingTests extends ESTestCase{ + public void testGenerateShardId() { int[][] possibleValues = new int[][] { {8,4,2}, {20, 10, 2}, {36, 12, 3}, {15,5,1} @@ -70,6 +69,7 @@ public void testGenerateShardId() { .numberOfReplicas(1) .setRoutingNumShards(shardSplits[0]).build(); int shrunkShard = OperationRouting.generateShardId(shrunk, term, null); + Set shardIds = IndexMetaData.selectShrinkShards(shrunkShard, metaData, shrunk.getNumberOfShards()); assertEquals(1, shardIds.stream().filter((sid) -> sid.id() == shard).count()); @@ -81,6 +81,36 @@ public void testGenerateShardId() { } } + public void testGenerateShardIdSplit() { + int[][] possibleValues = new int[][] { + {2,4,8}, {2, 10, 20}, {3, 12, 36}, {1,5,15} + }; + for (int i = 0; i < 10; i++) { + int[] shardSplits = randomFrom(possibleValues); + assertEquals(shardSplits[0], (shardSplits[0] * shardSplits[1]) / shardSplits[1]); + assertEquals(shardSplits[1], (shardSplits[1] * shardSplits[2]) / shardSplits[2]); + IndexMetaData metaData = IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(shardSplits[0]) + .numberOfReplicas(1).setRoutingNumShards(shardSplits[2]).build(); + String term = randomAlphaOfLength(10); + final int shard = OperationRouting.generateShardId(metaData, term, null); + IndexMetaData split = IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(shardSplits[1]) + .numberOfReplicas(1) + .setRoutingNumShards(shardSplits[2]).build(); + int shrunkShard = OperationRouting.generateShardId(split, term, null); + + ShardId shardId = IndexMetaData.selectSplitShard(shrunkShard, metaData, split.getNumberOfShards()); + assertNotNull(shardId); + assertEquals(shard, shardId.getId()); + + split = IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(shardSplits[2]).numberOfReplicas(1) + .setRoutingNumShards(shardSplits[2]).build(); + shrunkShard = OperationRouting.generateShardId(split, term, null); + shardId = IndexMetaData.selectSplitShard(shrunkShard, metaData, split.getNumberOfShards()); + assertNotNull(shardId); + assertEquals(shard, shardId.getId()); + } + } + public void testPartitionedIndex() { // make sure the same routing value always has each _id fall within the configured partition size for (int shards = 1; shards < 5; shards++) { @@ -373,7 +403,7 @@ public void testPreferNodes() throws InterruptedException, IOException { terminate(threadPool); } } - + public void testFairSessionIdPreferences() throws InterruptedException, IOException { // Ensure that a user session is re-routed back to same nodes for // subsequent searches and that the nodes are selected fairly i.e. @@ -424,13 +454,13 @@ public void testFairSessionIdPreferences() throws InterruptedException, IOExcept assertThat("Search should use more than one of the nodes", selectedNodes.size(), greaterThan(1)); } } - + // Regression test for the routing logic - implements same hashing logic private ShardIterator duelGetShards(ClusterState clusterState, ShardId shardId, String sessionId) { final IndexShardRoutingTable indexShard = clusterState.getRoutingTable().shardRoutingTable(shardId.getIndexName(), shardId.getId()); int routingHash = Murmur3HashFunction.hash(sessionId); routingHash = 31 * routingHash + indexShard.shardId.hashCode(); - return indexShard.activeInitializingShardsIt(routingHash); + return indexShard.activeInitializingShardsIt(routingHash); } public void testThatOnlyNodesSupportNodeIds() throws InterruptedException, IOException { diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ResizeAllocationDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ResizeAllocationDeciderTests.java new file mode 100644 index 0000000000000..cb9919216bdd0 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ResizeAllocationDeciderTests.java @@ -0,0 +1,287 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.cluster.routing.allocation; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ESAllocationTestCase; +import org.elasticsearch.cluster.EmptyClusterInfoService; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.RecoverySource; +import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.elasticsearch.cluster.routing.TestShardRouting; +import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator; +import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; +import org.elasticsearch.cluster.routing.allocation.decider.Decision; +import org.elasticsearch.cluster.routing.allocation.decider.ResizeAllocationDecider; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.test.VersionUtils; +import org.elasticsearch.test.gateway.TestGatewayAllocator; + +import java.util.Arrays; +import java.util.Collections; + +import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; +import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; +import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED; + + +public class ResizeAllocationDeciderTests extends ESAllocationTestCase { + + private AllocationService strategy; + + @Override + public void setUp() throws Exception { + super.setUp(); + strategy = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY, + Collections.singleton(new ResizeAllocationDecider(Settings.EMPTY))), + new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE); + } + + private ClusterState createInitialClusterState(boolean startShards) { + return createInitialClusterState(startShards, Version.CURRENT); + } + + private ClusterState createInitialClusterState(boolean startShards, Version nodeVersion) { + MetaData.Builder metaBuilder = MetaData.builder(); + metaBuilder.put(IndexMetaData.builder("source").settings(settings(Version.CURRENT)) + .numberOfShards(2).numberOfReplicas(0).setRoutingNumShards(16)); + MetaData metaData = metaBuilder.build(); + RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); + routingTableBuilder.addAsNew(metaData.index("source")); + + RoutingTable routingTable = routingTableBuilder.build(); + ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) + .metaData(metaData).routingTable(routingTable).build(); + clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1", nodeVersion)).add(newNode + ("node2", nodeVersion))) + .build(); + RoutingTable prevRoutingTable = routingTable; + routingTable = strategy.reroute(clusterState, "reroute", false).routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + + assertEquals(prevRoutingTable.index("source").shards().size(), 2); + assertEquals(prevRoutingTable.index("source").shard(0).shards().get(0).state(), UNASSIGNED); + assertEquals(prevRoutingTable.index("source").shard(1).shards().get(0).state(), UNASSIGNED); + + + assertEquals(routingTable.index("source").shards().size(), 2); + + assertEquals(routingTable.index("source").shard(0).shards().get(0).state(), INITIALIZING); + assertEquals(routingTable.index("source").shard(1).shards().get(0).state(), INITIALIZING); + + + if (startShards) { + clusterState = strategy.applyStartedShards(clusterState, + Arrays.asList(routingTable.index("source").shard(0).shards().get(0), + routingTable.index("source").shard(1).shards().get(0))); + routingTable = clusterState.routingTable(); + assertEquals(routingTable.index("source").shards().size(), 2); + assertEquals(routingTable.index("source").shard(0).shards().get(0).state(), STARTED); + assertEquals(routingTable.index("source").shard(1).shards().get(0).state(), STARTED); + + } + return clusterState; + } + + public void testNonResizeRouting() { + ClusterState clusterState = createInitialClusterState(true); + ResizeAllocationDecider resizeAllocationDecider = new ResizeAllocationDecider(Settings.EMPTY); + RoutingAllocation routingAllocation = new RoutingAllocation(null, null, clusterState, null, 0); + ShardRouting shardRouting = TestShardRouting.newShardRouting("non-resize", 0, null, true, ShardRoutingState.UNASSIGNED); + assertEquals(Decision.ALWAYS, resizeAllocationDecider.canAllocate(shardRouting, routingAllocation)); + assertEquals(Decision.ALWAYS, resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node1"), + routingAllocation)); + } + + public void testShrink() { // we don't handle shrink yet + ClusterState clusterState = createInitialClusterState(true); + MetaData.Builder metaBuilder = MetaData.builder(clusterState.metaData()); + metaBuilder.put(IndexMetaData.builder("target").settings(settings(Version.CURRENT) + .put(IndexMetaData.INDEX_RESIZE_SOURCE_NAME.getKey(), "source") + .put(IndexMetaData.INDEX_RESIZE_SOURCE_UUID_KEY, IndexMetaData.INDEX_UUID_NA_VALUE)) + .numberOfShards(1).numberOfReplicas(0)); + MetaData metaData = metaBuilder.build(); + RoutingTable.Builder routingTableBuilder = RoutingTable.builder(clusterState.routingTable()); + routingTableBuilder.addAsNew(metaData.index("target")); + + clusterState = ClusterState.builder(clusterState) + .routingTable(routingTableBuilder.build()) + .metaData(metaData).build(); + Index idx = clusterState.metaData().index("target").getIndex(); + + ResizeAllocationDecider resizeAllocationDecider = new ResizeAllocationDecider(Settings.EMPTY); + RoutingAllocation routingAllocation = new RoutingAllocation(null, null, clusterState, null, 0); + ShardRouting shardRouting = TestShardRouting.newShardRouting(new ShardId(idx, 0), null, true, RecoverySource + .LocalShardsRecoverySource.INSTANCE, ShardRoutingState.UNASSIGNED); + assertEquals(Decision.ALWAYS, resizeAllocationDecider.canAllocate(shardRouting, routingAllocation)); + assertEquals(Decision.ALWAYS, resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node1"), + routingAllocation)); + assertEquals(Decision.ALWAYS, resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node2"), + routingAllocation)); + } + + public void testSourceNotActive() { + ClusterState clusterState = createInitialClusterState(false); + MetaData.Builder metaBuilder = MetaData.builder(clusterState.metaData()); + metaBuilder.put(IndexMetaData.builder("target").settings(settings(Version.CURRENT) + .put(IndexMetaData.INDEX_RESIZE_SOURCE_NAME.getKey(), "source") + .put(IndexMetaData.INDEX_RESIZE_SOURCE_UUID_KEY, IndexMetaData.INDEX_UUID_NA_VALUE)) + .numberOfShards(4).numberOfReplicas(0)); + MetaData metaData = metaBuilder.build(); + RoutingTable.Builder routingTableBuilder = RoutingTable.builder(clusterState.routingTable()); + routingTableBuilder.addAsNew(metaData.index("target")); + + clusterState = ClusterState.builder(clusterState) + .routingTable(routingTableBuilder.build()) + .metaData(metaData).build(); + Index idx = clusterState.metaData().index("target").getIndex(); + + + ResizeAllocationDecider resizeAllocationDecider = new ResizeAllocationDecider(Settings.EMPTY); + RoutingAllocation routingAllocation = new RoutingAllocation(null, clusterState.getRoutingNodes(), clusterState, null, 0); + int shardId = randomIntBetween(0, 3); + int sourceShardId = IndexMetaData.selectSplitShard(shardId, clusterState.metaData().index("source"), 4).id(); + ShardRouting shardRouting = TestShardRouting.newShardRouting(new ShardId(idx, shardId), null, true, RecoverySource + .LocalShardsRecoverySource.INSTANCE, ShardRoutingState.UNASSIGNED); + assertEquals(Decision.NO, resizeAllocationDecider.canAllocate(shardRouting, routingAllocation)); + assertEquals(Decision.NO, resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node1"), + routingAllocation)); + assertEquals(Decision.NO, resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node2"), + routingAllocation)); + + routingAllocation.debugDecision(true); + assertEquals("source primary shard [[source][" + sourceShardId + "]] is not active", + resizeAllocationDecider.canAllocate(shardRouting, routingAllocation).getExplanation()); + assertEquals("source primary shard [[source][" + sourceShardId + "]] is not active", + resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node0"), + routingAllocation).getExplanation()); + assertEquals("source primary shard [[source][" + sourceShardId + "]] is not active", + resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node1"), + routingAllocation).getExplanation()); + } + + public void testSourcePrimaryActive() { + ClusterState clusterState = createInitialClusterState(true); + MetaData.Builder metaBuilder = MetaData.builder(clusterState.metaData()); + metaBuilder.put(IndexMetaData.builder("target").settings(settings(Version.CURRENT) + .put(IndexMetaData.INDEX_RESIZE_SOURCE_NAME.getKey(), "source") + .put(IndexMetaData.INDEX_RESIZE_SOURCE_UUID_KEY, IndexMetaData.INDEX_UUID_NA_VALUE)) + .numberOfShards(4).numberOfReplicas(0)); + MetaData metaData = metaBuilder.build(); + RoutingTable.Builder routingTableBuilder = RoutingTable.builder(clusterState.routingTable()); + routingTableBuilder.addAsNew(metaData.index("target")); + + clusterState = ClusterState.builder(clusterState) + .routingTable(routingTableBuilder.build()) + .metaData(metaData).build(); + Index idx = clusterState.metaData().index("target").getIndex(); + + + ResizeAllocationDecider resizeAllocationDecider = new ResizeAllocationDecider(Settings.EMPTY); + RoutingAllocation routingAllocation = new RoutingAllocation(null, clusterState.getRoutingNodes(), clusterState, null, 0); + int shardId = randomIntBetween(0, 3); + int sourceShardId = IndexMetaData.selectSplitShard(shardId, clusterState.metaData().index("source"), 4).id(); + ShardRouting shardRouting = TestShardRouting.newShardRouting(new ShardId(idx, shardId), null, true, RecoverySource + .LocalShardsRecoverySource.INSTANCE, ShardRoutingState.UNASSIGNED); + assertEquals(Decision.YES, resizeAllocationDecider.canAllocate(shardRouting, routingAllocation)); + + String allowedNode = clusterState.getRoutingTable().index("source").shard(sourceShardId).primaryShard().currentNodeId(); + + if ("node1".equals(allowedNode)) { + assertEquals(Decision.YES, resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node1"), + routingAllocation)); + assertEquals(Decision.NO, resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node2"), + routingAllocation)); + } else { + assertEquals(Decision.NO, resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node1"), + routingAllocation)); + assertEquals(Decision.YES, resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node2"), + routingAllocation)); + } + + routingAllocation.debugDecision(true); + assertEquals("source primary is active", resizeAllocationDecider.canAllocate(shardRouting, routingAllocation).getExplanation()); + + if ("node1".equals(allowedNode)) { + assertEquals("source primary is allocated on this node", + resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node1"), + routingAllocation).getExplanation()); + assertEquals("source primary is allocated on another node", + resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node2"), + routingAllocation).getExplanation()); + } else { + assertEquals("source primary is allocated on another node", + resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node1"), + routingAllocation).getExplanation()); + assertEquals("source primary is allocated on this node", + resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node2"), + routingAllocation).getExplanation()); + } + } + + public void testAllocateOnOldNode() { + Version version = VersionUtils.randomVersionBetween(random(), Version.V_5_0_0, + VersionUtils.getPreviousVersion(Version.V_7_0_0_alpha1)); + ClusterState clusterState = createInitialClusterState(true, version); + MetaData.Builder metaBuilder = MetaData.builder(clusterState.metaData()); + metaBuilder.put(IndexMetaData.builder("target").settings(settings(Version.CURRENT) + .put(IndexMetaData.INDEX_RESIZE_SOURCE_NAME.getKey(), "source") + .put(IndexMetaData.INDEX_RESIZE_SOURCE_UUID_KEY, IndexMetaData.INDEX_UUID_NA_VALUE)) + .numberOfShards(4).numberOfReplicas(0)); + MetaData metaData = metaBuilder.build(); + RoutingTable.Builder routingTableBuilder = RoutingTable.builder(clusterState.routingTable()); + routingTableBuilder.addAsNew(metaData.index("target")); + + clusterState = ClusterState.builder(clusterState) + .routingTable(routingTableBuilder.build()) + .metaData(metaData).build(); + Index idx = clusterState.metaData().index("target").getIndex(); + + + ResizeAllocationDecider resizeAllocationDecider = new ResizeAllocationDecider(Settings.EMPTY); + RoutingAllocation routingAllocation = new RoutingAllocation(null, clusterState.getRoutingNodes(), clusterState, null, 0); + int shardId = randomIntBetween(0, 3); + int sourceShardId = IndexMetaData.selectSplitShard(shardId, clusterState.metaData().index("source"), 4).id(); + ShardRouting shardRouting = TestShardRouting.newShardRouting(new ShardId(idx, shardId), null, true, RecoverySource + .LocalShardsRecoverySource.INSTANCE, ShardRoutingState.UNASSIGNED); + assertEquals(Decision.YES, resizeAllocationDecider.canAllocate(shardRouting, routingAllocation)); + + assertEquals(Decision.NO, resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node1"), + routingAllocation)); + assertEquals(Decision.NO, resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node2"), + routingAllocation)); + + routingAllocation.debugDecision(true); + assertEquals("source primary is active", resizeAllocationDecider.canAllocate(shardRouting, routingAllocation).getExplanation()); + assertEquals("node [node1] is too old to split a shard", + resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node1"), + routingAllocation).getExplanation()); + assertEquals("node [node2] is too old to split a shard", + resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node2"), + routingAllocation).getExplanation()); + } +} diff --git a/core/src/test/java/org/elasticsearch/index/shard/ShardSplittingQueryTests.java b/core/src/test/java/org/elasticsearch/index/shard/ShardSplittingQueryTests.java new file mode 100644 index 0000000000000..7351372620fc9 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/index/shard/ShardSplittingQueryTests.java @@ -0,0 +1,193 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.shard; + +import org.apache.lucene.document.Field; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Weight; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.routing.OperationRouting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.IdFieldMapper; +import org.elasticsearch.index.mapper.RoutingFieldMapper; +import org.elasticsearch.index.mapper.Uid; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +public class ShardSplittingQueryTests extends ESTestCase { + + public void testSplitOnID() throws IOException { + Directory dir = newFSDirectory(createTempDir()); + final int numDocs = randomIntBetween(50, 100); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + int numShards = randomIntBetween(2, 10); + IndexMetaData metaData = IndexMetaData.builder("test") + .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(numShards) + .setRoutingNumShards(numShards * 1000000) + .numberOfReplicas(0).build(); + int targetShardId = randomIntBetween(0, numShards-1); + for (int j = 0; j < numDocs; j++) { + int shardId = OperationRouting.generateShardId(metaData, Integer.toString(j), null); + writer.addDocument(Arrays.asList( + new StringField(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(j)), Field.Store.YES), + new SortedNumericDocValuesField("shard_id", shardId) + )); + } + writer.commit(); + writer.close(); + + + assertSplit(dir, metaData, targetShardId); + dir.close(); + } + + public void testSplitOnRouting() throws IOException { + Directory dir = newFSDirectory(createTempDir()); + final int numDocs = randomIntBetween(50, 100); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + int numShards = randomIntBetween(2, 10); + IndexMetaData metaData = IndexMetaData.builder("test") + .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(numShards) + .setRoutingNumShards(numShards * 1000000) + .numberOfReplicas(0).build(); + int targetShardId = randomIntBetween(0, numShards-1); + for (int j = 0; j < numDocs; j++) { + String routing = randomRealisticUnicodeOfCodepointLengthBetween(1, 5); + final int shardId = OperationRouting.generateShardId(metaData, null, routing); + writer.addDocument(Arrays.asList( + new StringField(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(j)), Field.Store.YES), + new StringField(RoutingFieldMapper.NAME, routing, Field.Store.YES), + new SortedNumericDocValuesField("shard_id", shardId) + )); + } + writer.commit(); + writer.close(); + assertSplit(dir, metaData, targetShardId); + dir.close(); + } + + public void testSplitOnIdOrRouting() throws IOException { + Directory dir = newFSDirectory(createTempDir()); + final int numDocs = randomIntBetween(50, 100); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + int numShards = randomIntBetween(2, 10); + IndexMetaData metaData = IndexMetaData.builder("test") + .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(numShards) + .setRoutingNumShards(numShards * 1000000) + .numberOfReplicas(0).build(); + int targetShardId = randomIntBetween(0, numShards-1); + for (int j = 0; j < numDocs; j++) { + if (randomBoolean()) { + String routing = randomRealisticUnicodeOfCodepointLengthBetween(1, 5); + final int shardId = OperationRouting.generateShardId(metaData, null, routing); + writer.addDocument(Arrays.asList( + new StringField(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(j)), Field.Store.YES), + new StringField(RoutingFieldMapper.NAME, routing, Field.Store.YES), + new SortedNumericDocValuesField("shard_id", shardId) + )); + } else { + int shardId = OperationRouting.generateShardId(metaData, Integer.toString(j), null); + writer.addDocument(Arrays.asList( + new StringField(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(j)), Field.Store.YES), + new SortedNumericDocValuesField("shard_id", shardId) + )); + } + } + writer.commit(); + writer.close(); + assertSplit(dir, metaData, targetShardId); + dir.close(); + } + + + public void testSplitOnRoutingPartitioned() throws IOException { + Directory dir = newFSDirectory(createTempDir()); + final int numDocs = randomIntBetween(50, 100); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + int numShards = randomIntBetween(2, 10); + IndexMetaData metaData = IndexMetaData.builder("test") + .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(numShards) + .setRoutingNumShards(numShards * 1000000) + .routingPartitionSize(randomIntBetween(1, 10)) + .numberOfReplicas(0).build(); + int targetShardId = randomIntBetween(0, numShards-1); + for (int j = 0; j < numDocs; j++) { + String routing = randomRealisticUnicodeOfCodepointLengthBetween(1, 5); + final int shardId = OperationRouting.generateShardId(metaData, Integer.toString(j), routing); + writer.addDocument(Arrays.asList( + new StringField(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(j)), Field.Store.YES), + new StringField(RoutingFieldMapper.NAME, routing, Field.Store.YES), + new SortedNumericDocValuesField("shard_id", shardId) + )); + } + writer.commit(); + writer.close(); + assertSplit(dir, metaData, targetShardId); + dir.close(); + } + + + + + void assertSplit(Directory dir, IndexMetaData metaData, int targetShardId) throws IOException { + try (IndexReader reader = DirectoryReader.open(dir)) { + IndexSearcher searcher = new IndexSearcher(reader); + searcher.setQueryCache(null); + final boolean needsScores = false; + final Weight splitWeight = searcher.createNormalizedWeight(new ShardSplittingQuery(metaData, targetShardId), needsScores); + final List leaves = reader.leaves(); + for (final LeafReaderContext ctx : leaves) { + Scorer scorer = splitWeight.scorer(ctx); + DocIdSetIterator iterator = scorer.iterator(); + SortedNumericDocValues shard_id = ctx.reader().getSortedNumericDocValues("shard_id"); + int doc; + while ((doc = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + while (shard_id.nextDoc() < doc) { + long shardID = shard_id.nextValue(); + assertEquals(shardID, targetShardId); + } + assertEquals(shard_id.docID(), doc); + long shardID = shard_id.nextValue(); + BytesRef id = reader.document(doc).getBinaryValue("_id"); + String actualId = Uid.decodeId(id.bytes, id.offset, id.length); + assertNotEquals(ctx.reader() + " docID: " + doc + " actualID: " + actualId, shardID, targetShardId); + } + } + } + } +} diff --git a/core/src/test/java/org/elasticsearch/index/shard/StoreRecoveryTests.java b/core/src/test/java/org/elasticsearch/index/shard/StoreRecoveryTests.java index 8d3ac8433d17d..05b092ff3a461 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/StoreRecoveryTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/StoreRecoveryTests.java @@ -25,17 +25,28 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.SegmentCommitInfo; import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.SortedNumericSortField; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.routing.OperationRouting; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.engine.InternalEngine; +import org.elasticsearch.index.mapper.IdFieldMapper; +import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.test.ESTestCase; @@ -46,7 +57,9 @@ import java.nio.file.attribute.BasicFileAttributes; import java.security.AccessControlException; import java.util.Arrays; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import java.util.function.Predicate; import static org.hamcrest.CoreMatchers.equalTo; @@ -87,7 +100,7 @@ public void testAddIndices() throws IOException { Directory target = newFSDirectory(createTempDir()); final long maxSeqNo = randomNonNegativeLong(); final long maxUnsafeAutoIdTimestamp = randomNonNegativeLong(); - storeRecovery.addIndices(indexStats, target, indexSort, dirs, maxSeqNo, maxUnsafeAutoIdTimestamp); + storeRecovery.addIndices(indexStats, target, indexSort, dirs, maxSeqNo, maxUnsafeAutoIdTimestamp, null, 0, false); int numFiles = 0; Predicate filesFilter = (f) -> f.startsWith("segments") == false && f.equals("write.lock") == false && f.startsWith("extra") == false; @@ -122,6 +135,99 @@ public void testAddIndices() throws IOException { IOUtils.close(dirs); } + public void testSplitShard() throws IOException { + Directory dir = newFSDirectory(createTempDir()); + final int numDocs = randomIntBetween(50, 100); + final Sort indexSort; + if (randomBoolean()) { + indexSort = new Sort(new SortedNumericSortField("num", SortField.Type.LONG, true)); + } else { + indexSort = null; + } + int id = 0; + IndexWriterConfig iwc = newIndexWriterConfig() + .setMergePolicy(NoMergePolicy.INSTANCE) + .setOpenMode(IndexWriterConfig.OpenMode.CREATE); + if (indexSort != null) { + iwc.setIndexSort(indexSort); + } + IndexWriter writer = new IndexWriter(dir, iwc); + for (int j = 0; j < numDocs; j++) { + writer.addDocument(Arrays.asList( + new StringField(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(j)), Field.Store.YES), + new SortedNumericDocValuesField("num", randomLong()) + )); + } + + writer.commit(); + writer.close(); + StoreRecovery storeRecovery = new StoreRecovery(new ShardId("foo", "bar", 1), logger); + RecoveryState.Index indexStats = new RecoveryState.Index(); + Directory target = newFSDirectory(createTempDir()); + final long maxSeqNo = randomNonNegativeLong(); + final long maxUnsafeAutoIdTimestamp = randomNonNegativeLong(); + int numShards = randomIntBetween(2, 10); + int targetShardId = randomIntBetween(0, numShards-1); + IndexMetaData metaData = IndexMetaData.builder("test") + .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(numShards) + .setRoutingNumShards(numShards * 1000000) + .numberOfReplicas(0).build(); + storeRecovery.addIndices(indexStats, target, indexSort, new Directory[] {dir}, maxSeqNo, maxUnsafeAutoIdTimestamp, metaData, + targetShardId, true); + + + SegmentInfos segmentCommitInfos = SegmentInfos.readLatestCommit(target); + final Map userData = segmentCommitInfos.getUserData(); + assertThat(userData.get(SequenceNumbers.MAX_SEQ_NO), equalTo(Long.toString(maxSeqNo))); + assertThat(userData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY), equalTo(Long.toString(maxSeqNo))); + assertThat(userData.get(InternalEngine.MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID), equalTo(Long.toString(maxUnsafeAutoIdTimestamp))); + for (SegmentCommitInfo info : segmentCommitInfos) { // check that we didn't merge + assertEquals("all sources must be flush", + info.info.getDiagnostics().get("source"), "flush"); + if (indexSort != null) { + assertEquals(indexSort, info.info.getIndexSort()); + } + } + + iwc = newIndexWriterConfig() + .setMergePolicy(NoMergePolicy.INSTANCE) + .setOpenMode(IndexWriterConfig.OpenMode.CREATE); + if (indexSort != null) { + iwc.setIndexSort(indexSort); + } + writer = new IndexWriter(target, iwc); + writer.forceMerge(1, true); + writer.commit(); + writer.close(); + + DirectoryReader reader = DirectoryReader.open(target); + for (LeafReaderContext ctx : reader.leaves()) { + LeafReader leafReader = ctx.reader(); + Terms terms = leafReader.terms(IdFieldMapper.NAME); + TermsEnum iterator = terms.iterator(); + BytesRef ref; + while((ref = iterator.next()) != null) { + String value = ref.utf8ToString(); + assertEquals("value has wrong shards: " + value, targetShardId, OperationRouting.generateShardId(metaData, value, null)); + } + for (int i = 0; i < numDocs; i++) { + ref = new BytesRef(Integer.toString(i)); + int shardId = OperationRouting.generateShardId(metaData, ref.utf8ToString(), null); + if (shardId == targetShardId) { + assertTrue(ref.utf8ToString() + " is missing", terms.iterator().seekExact(ref)); + } else { + assertFalse(ref.utf8ToString() + " was found but shouldn't", terms.iterator().seekExact(ref)); + } + + } + } + + reader.close(); + target.close(); + IOUtils.close(dir); + } + public void testStatsDirWrapper() throws IOException { Directory dir = newDirectory(); Directory target = newDirectory(); diff --git a/core/src/test/java/org/elasticsearch/routing/PartitionedRoutingIT.java b/core/src/test/java/org/elasticsearch/routing/PartitionedRoutingIT.java index b23ce6a9286bb..07a73a09f4ab4 100644 --- a/core/src/test/java/org/elasticsearch/routing/PartitionedRoutingIT.java +++ b/core/src/test/java/org/elasticsearch/routing/PartitionedRoutingIT.java @@ -105,7 +105,7 @@ public void testShrinking() throws Exception { index = "index_" + currentShards; logger.info("--> shrinking index [" + previousIndex + "] to [" + index + "]"); - client().admin().indices().prepareShrinkIndex(previousIndex, index) + client().admin().indices().prepareResizeIndex(previousIndex, index) .setSettings(Settings.builder() .put("index.number_of_shards", currentShards) .put("index.number_of_replicas", numberOfReplicas()) diff --git a/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java index 17412f8f724e4..5341b268544e7 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java +++ b/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java @@ -950,7 +950,7 @@ public void testRestoreShrinkIndex() throws Exception { logger.info("--> shrink the index"); assertAcked(client.admin().indices().prepareUpdateSettings(sourceIdx) .setSettings(Settings.builder().put("index.blocks.write", true)).get()); - assertAcked(client.admin().indices().prepareShrinkIndex(sourceIdx, shrunkIdx).get()); + assertAcked(client.admin().indices().prepareResizeIndex(sourceIdx, shrunkIdx).get()); logger.info("--> snapshot the shrunk index"); CreateSnapshotResponse createResponse = client.admin().cluster() diff --git a/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java b/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java index 6dd4fa384e99b..e5081481859ab 100644 --- a/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java +++ b/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java @@ -50,7 +50,7 @@ public class SharedSignificantTermsTestMethods { public static void aggregateAndCheckFromSeveralShards(ESIntegTestCase testCase) throws ExecutionException, InterruptedException { String type = ESTestCase.randomBoolean() ? "text" : "keyword"; - String settings = "{\"index.number_of_shards\": 5, \"index.number_of_replicas\": 0}"; + String settings = "{\"index.number_of_shards\": 7, \"index.number_of_replicas\": 0}"; index01Docs(type, settings, testCase); testCase.ensureGreen(); testCase.logClusterState(); diff --git a/docs/reference/indices.asciidoc b/docs/reference/indices.asciidoc index 873021c420636..70d3b19d3fd59 100644 --- a/docs/reference/indices.asciidoc +++ b/docs/reference/indices.asciidoc @@ -16,6 +16,7 @@ index settings, aliases, mappings, and index templates. * <> * <> * <> +* <> * <> [float] diff --git a/docs/reference/indices/split-index.asciidoc b/docs/reference/indices/split-index.asciidoc new file mode 100644 index 0000000000000..467c09baa2432 --- /dev/null +++ b/docs/reference/indices/split-index.asciidoc @@ -0,0 +1,165 @@ +[[indices-split-index]] +== Split Index + +number_of_routing_shards + +The split index API allows you to split an existing index into a new index +with multiple of it's primary shards. Similarly to the <> +where the number of primary shards in the shrunk index must be a factor of the source index. +The `_split` API requires the source index to be created with a specific number of routing shards +in order to be split in the future. (Note: this requirement might be remove in future releases) +The number of routing shards specify the hashing space that is used internally to distribute documents +across shards, in oder to have a consistent hashing that is compatible with the method elasticsearch +uses today. +For example an index with `8` primary shards and a `index.number_of_routing_shards` of `32` +can be split into `16` and `32` primary shards. An index with `1` primary shard +and `index.number_of_routing_shards` of `64` can be split into `2`, `4`, `8`, `16`, `32` or `64`. +The same works for non power of two routing shards ie. an index with `1` primary shard and +`index.number_of_routing_shards` set to `15` can be split into `3` and `15` or alternatively`5` and `15`. +The number of shards in the split index must always be a factor of `index.number_of_routing_shards` +in the source index. Before splitting, a (primary) copy of every shard in the index must be active in the cluster. + +Splitting works as follows: + +* First, it creates a new target index with the same definition as the source + index, but with a larger number of primary shards. + +* Then it hard-links segments from the source index into the target index. (If + the file system doesn't support hard-linking, then all segments are copied + into the new index, which is a much more time consuming process.) + +* Once the low level files are created all documents will be `hashed` again to delete + documents that belong in a different shard. + +* Finally, it recovers the target index as though it were a closed index which + had just been re-opened. + +[float] +=== Preparing an index for splitting + +Create an index with a routing shards factor: + +[source,js] +-------------------------------------------------- +PUT my_source_index +{ + "settings": { + "index.number_of_shards" : 1, + "index.number_of_routing_shards" : 2 <1> + } +} +------------------------------------------------- +// CONSOLE + +<1> Allows to split the index into two shards or in other words, it allows + for a single split operation. + +In order to split an index, the index must be marked as read-only, +and have <> `green`. + +This can be achieved with the following request: + +[source,js] +-------------------------------------------------- +PUT /my_source_index/_settings +{ + "settings": { + "index.blocks.write": true <1> + } +} +-------------------------------------------------- +// CONSOLE +// TEST[continued] + +<1> Prevents write operations to this index while still allowing metadata + changes like deleting the index. + +[float] +=== Spitting an index + +To split `my_source_index` into a new index called `my_target_index`, issue +the following request: + +[source,js] +-------------------------------------------------- +POST my_source_index/_split/my_target_index +{ + "settings": { + "index.number_of_shards": 2 + } +} +-------------------------------------------------- +// CONSOLE +// TEST[continued] + +The above request returns immediately once the target index has been added to +the cluster state -- it doesn't wait for the split operation to start. + +[IMPORTANT] +===================================== + +Indices can only be split if they satisfy the following requirements: + +* the target index must not exist + +* The index must have less primary shards than the target index. + +* The number of primary shards in the target index must be a factor of the + number of primary shards in the source index. + +* The node handling the split process must have sufficient free disk space to + accommodate a second copy of the existing index. + +===================================== + +The `_split` API is similar to the <> +and accepts `settings` and `aliases` parameters for the target index: + +[source,js] +-------------------------------------------------- +POST my_source_index/_split/my_target_index +{ + "settings": { + "index.number_of_shards": 5 <1> + }, + "aliases": { + "my_search_indices": {} + } +} +-------------------------------------------------- +// CONSOLE +// TEST[s/^/PUT my_source_index\n{"settings": {"index.blocks.write": true, "index.number_of_routing_shards" : 5, "index.number_of_shards": "1"}}\n/] + +<1> The number of shards in the target index. This must be a factor of the + number of shards in the source index. + + +NOTE: Mappings may not be specified in the `_split` request, and all +`index.analysis.*` and `index.similarity.*` settings will be overwritten with +the settings from the source index. + +[float] +=== Monitoring the split process + +The split process can be monitored with the <>, or the <> can be used to wait +until all primary shards have been allocated by setting the `wait_for_status` +parameter to `yellow`. + +The `_split` API returns as soon as the target index has been added to the +cluster state, before any shards have been allocated. At this point, all +shards are in the state `unassigned`. If, for any reason, the target index +can't be allocated, its primary shard will remain `unassigned` until it +can be allocated on that node. + +Once the primary shard is allocated, it moves to state `initializing`, and the +split process begins. When the split operation completes, the shard will +become `active`. At that point, Elasticsearch will try to allocate any +replicas and may decide to relocate the primary shard to another node. + +[float] +=== Wait For Active Shards + +Because the split operation creates a new index to split the shards to, +the <> setting +on index creation applies to the split index action as well. diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.split.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.split.json new file mode 100644 index 0000000000000..a79fa7b708269 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.split.json @@ -0,0 +1,39 @@ +{ + "indices.split": { + "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-split-index.html", + "methods": ["PUT", "POST"], + "url": { + "path": "/{index}/_split/{target}", + "paths": ["/{index}/_split/{target}"], + "parts": { + "index": { + "type" : "string", + "required" : true, + "description" : "The name of the source index to split" + }, + "target": { + "type" : "string", + "required" : true, + "description" : "The name of the target index to split into" + } + }, + "params": { + "timeout": { + "type" : "time", + "description" : "Explicit operation timeout" + }, + "master_timeout": { + "type" : "time", + "description" : "Specify timeout for connection to master" + }, + "wait_for_active_shards": { + "type" : "string", + "description" : "Set the number of active shards to wait for on the shrunken index before the operation returns." + } + } + }, + "body": { + "description" : "The configuration for the target index (`settings` and `aliases`)" + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml new file mode 100644 index 0000000000000..f51fc808b4623 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml @@ -0,0 +1,101 @@ +--- +"Split index via API": + - skip: + version: " - 6.99.99" + reason: Added in 7.0.0 + - do: + indices.create: + index: source + wait_for_active_shards: 1 + body: + settings: + index.number_of_shards: 1 + index.number_of_replicas: 0 + index.number_of_routing_shards: 2 + - do: + index: + index: source + type: doc + id: "1" + body: { "foo": "hello world" } + + - do: + index: + index: source + type: doc + id: "2" + body: { "foo": "hello world 2" } + + - do: + index: + index: source + type: doc + id: "3" + body: { "foo": "hello world 3" } + + # make it read-only + - do: + indices.put_settings: + index: source + body: + index.blocks.write: true + index.number_of_replicas: 0 + + - do: + cluster.health: + wait_for_status: green + index: source + + # now we do the actual split + - do: + indices.split: + index: "source" + target: "target" + wait_for_active_shards: 1 + master_timeout: 10s + body: + settings: + index.number_of_replicas: 0 + index.number_of_shards: 2 + + - do: + cluster.health: + wait_for_status: green + + - do: + get: + index: target + type: doc + id: "1" + + - match: { _index: target } + - match: { _type: doc } + - match: { _id: "1" } + - match: { _source: { foo: "hello world" } } + + + - do: + get: + index: target + type: doc + id: "2" + + - match: { _index: target } + - match: { _type: doc } + - match: { _id: "2" } + - match: { _source: { foo: "hello world 2" } } + + + - do: + get: + index: target + type: doc + id: "3" + + - match: { _index: target } + - match: { _type: doc } + - match: { _id: "3" } + - match: { _source: { foo: "hello world 3" } } + + + diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/20_source_mapping.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/20_source_mapping.yml new file mode 100644 index 0000000000000..ffd7ffe7a2946 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/20_source_mapping.yml @@ -0,0 +1,72 @@ +--- +"Split index ignores target template mapping": + - skip: + version: " - 6.99.99" + reason: added in 7.0.0 + + # create index + - do: + indices.create: + index: source + wait_for_active_shards: 1 + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + index.number_of_routing_shards: 2 + mappings: + test: + properties: + count: + type: text + + # index document + - do: + index: + index: source + type: test + id: "1" + body: { "count": "1" } + + # create template matching shrink target + - do: + indices.put_template: + name: tpl1 + body: + index_patterns: targ* + mappings: + test: + properties: + count: + type: integer + + # make it read-only + - do: + indices.put_settings: + index: source + body: + index.blocks.write: true + index.number_of_replicas: 0 + + - do: + cluster.health: + wait_for_status: green + index: source + + # now we do the actual split + - do: + indices.split: + index: "source" + target: "target" + wait_for_active_shards: 1 + master_timeout: 10s + body: + settings: + index.number_of_shards: 2 + index.number_of_replicas: 0 + + - do: + cluster.health: + wait_for_status: green + + diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java b/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java index a4ac6fad241a5..2291c3d39e200 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java @@ -39,6 +39,10 @@ public static ShardRouting newShardRouting(String index, int shardId, String cur return newShardRouting(new ShardId(index, IndexMetaData.INDEX_UUID_NA_VALUE, shardId), currentNodeId, primary, state); } + public static ShardRouting newShardRouting(ShardId shardId, String currentNodeId, boolean primary, RecoverySource recoverySource, ShardRoutingState state) { + return new ShardRouting(shardId, currentNodeId, null, primary, state, recoverySource, buildUnassignedInfo(state), buildAllocationId(state), -1); + } + public static ShardRouting newShardRouting(ShardId shardId, String currentNodeId, boolean primary, ShardRoutingState state) { return new ShardRouting(shardId, currentNodeId, null, primary, state, buildRecoveryTarget(primary, state), buildUnassignedInfo(state), buildAllocationId(state), -1); }