diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java b/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java
index d004e34d06efd..fa7bff1435b12 100644
--- a/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java
+++ b/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java
@@ -21,6 +21,7 @@
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+import org.elasticsearch.Version;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.io.stream.StreamInput;
@@ -29,19 +30,26 @@
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.shard.ShardId;
import java.io.IOException;
import java.util.Map;
+import java.util.stream.StreamSupport;
+
/**
* ClusterInfo is an object representing a map of nodes to {@link DiskUsage}
- * and a map of shard ids to shard sizes, see
- * InternalClusterInfoService.shardIdentifierFromRouting(String)
- * for the key used in the shardSizes map
+ * and a map of shard ids to shard sizes.
*/
public class ClusterInfo implements ToXContentFragment, Writeable {
+
+ /**
+ * Only needed to support shard sizes keyed by String as in {@link Version#V_7_0_0} - TODO remove this in v9.
+ */
+ private static final String BWC_SHARD_ID_UUID = "_bwc_shard_id_uuid_";
+
private final ImmutableOpenMap leastAvailableSpaceUsage;
private final ImmutableOpenMap mostAvailableSpaceUsage;
- final ImmutableOpenMap shardSizes;
+ final ImmutableOpenMap shardSizes;
public static final ClusterInfo EMPTY = new ClusterInfo();
final ImmutableOpenMap routingToDataPath;
@@ -56,10 +64,9 @@ protected ClusterInfo() {
* @param mostAvailableSpaceUsage a node id to disk usage mapping for the path that has the most available space on the node.
* @param shardSizes a shardkey to size in bytes mapping per shard.
* @param routingToDataPath the shard routing to datapath mapping
- * @see #shardIdentifierFromRouting
*/
public ClusterInfo(ImmutableOpenMap leastAvailableSpaceUsage,
- ImmutableOpenMap mostAvailableSpaceUsage, ImmutableOpenMap shardSizes,
+ ImmutableOpenMap mostAvailableSpaceUsage, ImmutableOpenMap shardSizes,
ImmutableOpenMap routingToDataPath) {
this.leastAvailableSpaceUsage = leastAvailableSpaceUsage;
this.shardSizes = shardSizes;
@@ -67,17 +74,29 @@ public ClusterInfo(ImmutableOpenMap leastAvailableSpaceUsage,
this.routingToDataPath = routingToDataPath;
}
+ private static ShardId readShardId(StreamInput in) throws IOException {
+ if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+ return new ShardId(in);
+ } else {
+ // earlier versions used a string of the form "[indexname][0][p]" or "[indexname][0][r]" to distinguish primaries from replicas
+ // but the only time we use a serialized ClusterInfo is for allocation explanation, not for allocation calculations, so we can
+ // use a placeholder with a fake UUID and shard number and use the index name field for the original string:
+ final String shardKey = in.readString();
+ return new ShardId(shardKey, BWC_SHARD_ID_UUID, 0);
+ }
+ }
+
public ClusterInfo(StreamInput in) throws IOException {
Map leastMap = in.readMap(StreamInput::readString, DiskUsage::new);
Map mostMap = in.readMap(StreamInput::readString, DiskUsage::new);
- Map sizeMap = in.readMap(StreamInput::readString, StreamInput::readLong);
+ Map sizeMap = in.readMap(ClusterInfo::readShardId, StreamInput::readLong);
Map routingMap = in.readMap(ShardRouting::new, StreamInput::readString);
ImmutableOpenMap.Builder leastBuilder = ImmutableOpenMap.builder();
this.leastAvailableSpaceUsage = leastBuilder.putAll(leastMap).build();
ImmutableOpenMap.Builder mostBuilder = ImmutableOpenMap.builder();
this.mostAvailableSpaceUsage = mostBuilder.putAll(mostMap).build();
- ImmutableOpenMap.Builder sizeBuilder = ImmutableOpenMap.builder();
+ ImmutableOpenMap.Builder sizeBuilder = ImmutableOpenMap.builder();
this.shardSizes = sizeBuilder.putAll(sizeMap).build();
ImmutableOpenMap.Builder routingBuilder = ImmutableOpenMap.builder();
this.routingToDataPath = routingBuilder.putAll(routingMap).build();
@@ -96,8 +115,12 @@ public void writeTo(StreamOutput out) throws IOException {
c.value.writeTo(out);
}
out.writeVInt(this.shardSizes.size());
- for (ObjectObjectCursor c : this.shardSizes) {
- out.writeString(c.key);
+ for (ObjectObjectCursor c : this.shardSizes) {
+ if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+ c.key.writeTo(out);
+ } else {
+ out.writeString(c.key.toString());
+ }
if (c.value == null) {
out.writeLong(-1);
} else {
@@ -133,8 +156,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
}
builder.endObject(); // end "nodes"
builder.startObject("shard_sizes"); {
- for (ObjectObjectCursor c : this.shardSizes) {
- builder.humanReadableField(c.key + "_bytes", c.key, new ByteSizeValue(c.value));
+ for (ObjectObjectCursor c : this.shardSizes) {
+ final String shardKey;
+ if (BWC_SHARD_ID_UUID.equals(c.key.getIndex().getUUID())) {
+ shardKey = c.key.getIndexName();
+ } else {
+ shardKey = c.key.toString();
+ }
+ builder.humanReadableField(shardKey + "_bytes", shardKey, new ByteSizeValue(c.value));
}
}
builder.endObject(); // end "shard_sizes"
@@ -165,7 +194,10 @@ public ImmutableOpenMap getNodeMostAvailableDiskUsages() {
* Returns the shard size for the given shard routing or null
it that metric is not available.
*/
public Long getShardSize(ShardRouting shardRouting) {
- return shardSizes.get(shardIdentifierFromRouting(shardRouting));
+ // this ClusterInfo instance was never serialized and sent over the wire, so it does not have any fake shard IDs.
+ assert StreamSupport.stream(shardSizes.keys().spliterator(), false)
+ .noneMatch(c -> BWC_SHARD_ID_UUID.equals(c.value.getIndex().getUUID())) : shardSizes;
+ return shardSizes.get(shardRouting.shardId());
}
/**
@@ -183,11 +215,4 @@ public long getShardSize(ShardRouting shardRouting, long defaultValue) {
return shardSize == null ? defaultValue : shardSize;
}
- /**
- * Method that incorporates the ShardId for the shard into a string that
- * includes a 'p' or 'r' depending on whether the shard is a primary.
- */
- static String shardIdentifierFromRouting(ShardRouting shardRouting) {
- return shardRouting.shardId().toString() + "[" + (shardRouting.primary() ? "p" : "r") + "]";
- }
}
diff --git a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java
index d07897199ae8e..b24f1abc060b3 100644
--- a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java
@@ -43,6 +43,7 @@
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.monitor.fs.FsInfo;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.ReceiveTimeoutTransportException;
@@ -80,7 +81,7 @@ public class InternalClusterInfoService implements ClusterInfoService, LocalNode
private volatile ImmutableOpenMap leastAvailableSpaceUsages;
private volatile ImmutableOpenMap mostAvailableSpaceUsages;
private volatile ImmutableOpenMap shardRoutingToDataPath;
- private volatile ImmutableOpenMap shardSizes;
+ private volatile ImmutableOpenMap shardSizes;
private volatile boolean isMaster = false;
private volatile boolean enabled;
private volatile TimeValue fetchTimeout;
@@ -320,7 +321,7 @@ public void onFailure(Exception e) {
@Override
public void onResponse(IndicesStatsResponse indicesStatsResponse) {
ShardStats[] stats = indicesStatsResponse.getShards();
- ImmutableOpenMap.Builder newShardSizes = ImmutableOpenMap.builder();
+ ImmutableOpenMap.Builder newShardSizes = ImmutableOpenMap.builder();
ImmutableOpenMap.Builder newShardRoutingToDataPath = ImmutableOpenMap.builder();
buildShardLevelInfo(logger, stats, newShardSizes, newShardRoutingToDataPath);
shardSizes = newShardSizes.build();
@@ -381,16 +382,17 @@ public void addListener(Consumer clusterInfoConsumer) {
listeners.add(clusterInfoConsumer);
}
- static void buildShardLevelInfo(Logger logger, ShardStats[] stats, ImmutableOpenMap.Builder newShardSizes,
- ImmutableOpenMap.Builder newShardRoutingToDataPath) {
- for (ShardStats s : stats) {
- newShardRoutingToDataPath.put(s.getShardRouting(), s.getDataPath());
- long size = s.getStats().getStore().sizeInBytes();
- String sid = ClusterInfo.shardIdentifierFromRouting(s.getShardRouting());
- if (logger.isTraceEnabled()) {
- logger.trace("shard: {} size: {}", sid, size);
- }
- newShardSizes.put(sid, size);
+ static void buildShardLevelInfo(Logger logger, ShardStats[] shardStatses,
+ ImmutableOpenMap.Builder shardSizes,
+ ImmutableOpenMap.Builder shardRoutingToDataPath) {
+ for (ShardStats shardStats : shardStatses) {
+ shardRoutingToDataPath.put(shardStats.getShardRouting(), shardStats.getDataPath());
+ final ShardId shardId = shardStats.getShardRouting().shardId();
+ final Long oldShardSize = shardSizes.get(shardId);
+ final long newShardSize = shardStats.getStats().getStore().sizeInBytes();
+ logger.trace("shard: {} size: {}", shardStats.getShardRouting(), newShardSize);
+ final long maxShardSize = oldShardSize == null ? newShardSize : Math.max(oldShardSize, newShardSize);
+ shardSizes.put(shardId, maxShardSize);
}
}
diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java b/server/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java
index aa897f10bb895..c49deabe043d3 100644
--- a/server/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java
+++ b/server/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java
@@ -38,6 +38,7 @@
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.store.Store;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.plugins.ActionPlugin;
@@ -130,7 +131,7 @@ public void testClusterInfoServiceCollectsInformation() throws Exception {
assertNotNull("info should not be null", info);
ImmutableOpenMap leastUsages = info.getNodeLeastAvailableDiskUsages();
ImmutableOpenMap mostUsages = info.getNodeMostAvailableDiskUsages();
- ImmutableOpenMap shardSizes = info.shardSizes;
+ ImmutableOpenMap shardSizes = info.shardSizes;
assertNotNull(leastUsages);
assertNotNull(shardSizes);
assertThat("some usages are populated", leastUsages.values().size(), Matchers.equalTo(2));
@@ -209,7 +210,7 @@ public void testClusterInfoServiceInformationClearOnError() {
assertThat(info.getNodeLeastAvailableDiskUsages().size(), greaterThanOrEqualTo(1));
assertThat(info.getNodeMostAvailableDiskUsages().size(), greaterThanOrEqualTo(1));
// indices is guaranteed to time out on the latch, not updating anything.
- assertThat(info.shardSizes.size(), greaterThan(1));
+ assertThat(info.shardSizes.size(), greaterThan(0));
// now we cause an exception
timeout.set(false);
diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java
index 35a7ea2aab80c..aaa787504b232 100644
--- a/server/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java
@@ -21,6 +21,7 @@
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.ShardRoutingState;
import org.elasticsearch.cluster.routing.TestShardRouting;
+import org.elasticsearch.common.UUIDs;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.index.shard.ShardId;
@@ -56,13 +57,12 @@ private static ImmutableOpenMap randomDiskUsage() {
return builder.build();
}
- private static ImmutableOpenMap randomShardSizes() {
+ private static ImmutableOpenMap randomShardSizes() {
int numEntries = randomIntBetween(0, 128);
- ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(numEntries);
+ ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(numEntries);
for (int i = 0; i < numEntries; i++) {
- String key = randomAlphaOfLength(32);
long shardSize = randomIntBetween(0, Integer.MAX_VALUE);
- builder.put(key, shardSize);
+ builder.put(new ShardId(randomAlphaOfLength(10), UUIDs.randomBase64UUID(random()), between(0, 100)), shardSize);
}
return builder.build();
}
diff --git a/server/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java b/server/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
index 55eae6fc0e95f..cd36d4cdbd199 100644
--- a/server/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
@@ -116,15 +116,15 @@ public void testFillShardLevelInfo() {
new ShardStats(test_0, new ShardPath(false, test0Path, test0Path, test_0.shardId()), commonStats0 , null, null, null),
new ShardStats(test_1, new ShardPath(false, test1Path, test1Path, test_1.shardId()), commonStats1 , null, null, null)
};
- ImmutableOpenMap.Builder shardSizes = ImmutableOpenMap.builder();
+ ImmutableOpenMap.Builder shardSizes = ImmutableOpenMap.builder();
ImmutableOpenMap.Builder routingToPath = ImmutableOpenMap.builder();
ClusterState state = ClusterState.builder(new ClusterName("blarg")).version(0).build();
InternalClusterInfoService.buildShardLevelInfo(logger, stats, shardSizes, routingToPath);
assertEquals(2, shardSizes.size());
- assertTrue(shardSizes.containsKey(ClusterInfo.shardIdentifierFromRouting(test_0)));
- assertTrue(shardSizes.containsKey(ClusterInfo.shardIdentifierFromRouting(test_1)));
- assertEquals(100L, shardSizes.get(ClusterInfo.shardIdentifierFromRouting(test_0)).longValue());
- assertEquals(1000L, shardSizes.get(ClusterInfo.shardIdentifierFromRouting(test_1)).longValue());
+ assertTrue(shardSizes.containsKey(test_0.shardId()));
+ assertTrue(shardSizes.containsKey(test_1.shardId()));
+ assertEquals(100L, shardSizes.get(test_0.shardId()).longValue());
+ assertEquals(1000L, shardSizes.get(test_1.shardId()).longValue());
assertEquals(2, routingToPath.size());
assertTrue(routingToPath.containsKey(test_0));
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
index d0a79a5a7834c..6c3dc527b6ef2 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
@@ -50,6 +50,7 @@
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.test.gateway.TestGatewayAllocator;
import java.util.Arrays;
@@ -71,10 +72,14 @@
public class DiskThresholdDeciderTests extends ESAllocationTestCase {
- DiskThresholdDecider makeDecider(Settings settings) {
+ private DiskThresholdDecider makeDecider(Settings settings) {
return new DiskThresholdDecider(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
}
+ private static ShardId shardId(String index, int shardNum) {
+ return new ShardId(index, index, shardNum);
+ }
+
public void testDiskThreshold() {
Settings diskSettings = Settings.builder()
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), true)
@@ -89,10 +94,9 @@ public void testDiskThreshold() {
usagesBuilder.put("node4", new DiskUsage("node4", "node4", "/dev/null", 100, 80)); // 20% used
ImmutableOpenMap usages = usagesBuilder.build();
- ImmutableOpenMap.Builder shardSizesBuilder = ImmutableOpenMap.builder();
- shardSizesBuilder.put("[test][0][p]", 10L); // 10 bytes
- shardSizesBuilder.put("[test][0][r]", 10L);
- ImmutableOpenMap shardSizes = shardSizesBuilder.build();
+ ImmutableOpenMap.Builder shardSizesBuilder = ImmutableOpenMap.builder();
+ shardSizesBuilder.put(shardId("test", 0), 10L); // 10 bytes
+ ImmutableOpenMap shardSizes = shardSizesBuilder.build();
final ClusterInfo clusterInfo = new DevNullClusterInfo(usages, usages, shardSizes);
ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
@@ -109,7 +113,7 @@ public void testDiskThreshold() {
new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), cis);
MetaData metaData = MetaData.builder()
- .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
+ .put(indexMetaData("test", 1, 1))
.build();
final RoutingTable initialRoutingTable = RoutingTable.builder()
@@ -264,10 +268,9 @@ public void testDiskThresholdWithAbsoluteSizes() {
usagesBuilder.put("node5", new DiskUsage("node5", "n5", "/dev/null", 100, 85)); // 15% used
ImmutableOpenMap usages = usagesBuilder.build();
- ImmutableOpenMap.Builder shardSizesBuilder = ImmutableOpenMap.builder();
- shardSizesBuilder.put("[test][0][p]", 10L); // 10 bytes
- shardSizesBuilder.put("[test][0][r]", 10L);
- ImmutableOpenMap shardSizes = shardSizesBuilder.build();
+ ImmutableOpenMap.Builder shardSizesBuilder = ImmutableOpenMap.builder();
+ shardSizesBuilder.put(shardId("test", 0), 10L); // 10 bytes
+ ImmutableOpenMap shardSizes = shardSizesBuilder.build();
final ClusterInfo clusterInfo = new DevNullClusterInfo(usages, usages, shardSizes);
ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
@@ -285,7 +288,7 @@ public void testDiskThresholdWithAbsoluteSizes() {
new BalancedShardsAllocator(Settings.EMPTY), cis);
MetaData metaData = MetaData.builder()
- .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(2))
+ .put(indexMetaData("test", 1, 2))
.build();
RoutingTable initialRoutingTable = RoutingTable.builder()
@@ -496,9 +499,9 @@ public void testDiskThresholdWithShardSizes() {
usagesBuilder.put("node2", new DiskUsage("node2", "n2", "/dev/null", 100, 1)); // 99% used
ImmutableOpenMap usages = usagesBuilder.build();
- ImmutableOpenMap.Builder shardSizesBuilder = ImmutableOpenMap.builder();
- shardSizesBuilder.put("[test][0][p]", 10L); // 10 bytes
- ImmutableOpenMap shardSizes = shardSizesBuilder.build();
+ ImmutableOpenMap.Builder shardSizesBuilder = ImmutableOpenMap.builder();
+ shardSizesBuilder.put(shardId("test", 0), 10L); // 10 bytes
+ ImmutableOpenMap shardSizes = shardSizesBuilder.build();
final ClusterInfo clusterInfo = new DevNullClusterInfo(usages, usages, shardSizes);
AllocationDeciders deciders = new AllocationDeciders(
@@ -517,7 +520,7 @@ Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLU
new BalancedShardsAllocator(Settings.EMPTY), cis);
MetaData metaData = MetaData.builder()
- .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0))
+ .put(indexMetaData("test", 1, 0))
.build();
RoutingTable routingTable = RoutingTable.builder()
@@ -555,10 +558,9 @@ public void testUnknownDiskUsage() {
usagesBuilder.put("node3", new DiskUsage("node3", "node3", "/dev/null", 100, 0)); // 100% used
ImmutableOpenMap usages = usagesBuilder.build();
- ImmutableOpenMap.Builder shardSizesBuilder = ImmutableOpenMap.builder();
- shardSizesBuilder.put("[test][0][p]", 10L); // 10 bytes
- shardSizesBuilder.put("[test][0][r]", 10L); // 10 bytes
- ImmutableOpenMap shardSizes = shardSizesBuilder.build();
+ ImmutableOpenMap.Builder shardSizesBuilder = ImmutableOpenMap.builder();
+ shardSizesBuilder.put(shardId("test", 0), 10L); // 10 bytes
+ ImmutableOpenMap shardSizes = shardSizesBuilder.build();
final ClusterInfo clusterInfo = new DevNullClusterInfo(usages, usages, shardSizes);
AllocationDeciders deciders = new AllocationDeciders(
@@ -577,7 +579,7 @@ Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLU
new BalancedShardsAllocator(Settings.EMPTY), cis);
MetaData metaData = MetaData.builder()
- .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0))
+ .put(indexMetaData("test", 1, 0))
.build();
RoutingTable routingTable = RoutingTable.builder()
@@ -647,12 +649,10 @@ public void testShardRelocationsTakenIntoAccount() {
usagesBuilder.put("node3", new DiskUsage("node3", "n3", "/dev/null", 100, 40)); // 60% used
ImmutableOpenMap usages = usagesBuilder.build();
- ImmutableOpenMap.Builder shardSizesBuilder = ImmutableOpenMap.builder();
- shardSizesBuilder.put("[test][0][p]", 14L); // 14 bytes
- shardSizesBuilder.put("[test][0][r]", 14L);
- shardSizesBuilder.put("[test2][0][p]", 1L); // 1 bytes
- shardSizesBuilder.put("[test2][0][r]", 1L);
- ImmutableOpenMap shardSizes = shardSizesBuilder.build();
+ ImmutableOpenMap.Builder shardSizesBuilder = ImmutableOpenMap.builder();
+ shardSizesBuilder.put(shardId("test", 0), 14L); // 14 bytes
+ shardSizesBuilder.put(shardId("test2", 0), 1L); // 1 bytes
+ ImmutableOpenMap shardSizes = shardSizesBuilder.build();
final ClusterInfo clusterInfo = new DevNullClusterInfo(usages, usages, shardSizes);
DiskThresholdDecider decider = makeDecider(diskSettings);
@@ -671,8 +671,8 @@ public void testShardRelocationsTakenIntoAccount() {
new BalancedShardsAllocator(Settings.EMPTY), cis);
MetaData metaData = MetaData.builder()
- .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
- .put(IndexMetaData.builder("test2").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
+ .put(indexMetaData("test", 1, 1))
+ .put(indexMetaData("test2", 1, 1))
.build();
RoutingTable initialRoutingTable = RoutingTable.builder()
@@ -720,12 +720,10 @@ public void testShardRelocationsTakenIntoAccount() {
overfullUsagesBuilder.put("node3", new DiskUsage("node3", "n3", "/dev/null", 100, 0)); // 100% used
final ImmutableOpenMap overfullUsages = overfullUsagesBuilder.build();
- final ImmutableOpenMap.Builder largerShardSizesBuilder = ImmutableOpenMap.builder();
- largerShardSizesBuilder.put("[test][0][p]", 14L);
- largerShardSizesBuilder.put("[test][0][r]", 14L);
- largerShardSizesBuilder.put("[test2][0][p]", 2L);
- largerShardSizesBuilder.put("[test2][0][r]", 2L);
- final ImmutableOpenMap largerShardSizes = largerShardSizesBuilder.build();
+ final ImmutableOpenMap.Builder largerShardSizesBuilder = ImmutableOpenMap.builder();
+ largerShardSizesBuilder.put(shardId("test", 0), 14L); // 14 bytes
+ largerShardSizesBuilder.put(shardId("test2", 0), 2L); // 2 bytes
+ final ImmutableOpenMap largerShardSizes = largerShardSizesBuilder.build();
final ClusterInfo overfullClusterInfo = new DevNullClusterInfo(overfullUsages, overfullUsages, largerShardSizes);
@@ -776,18 +774,18 @@ public void testCanRemainWithShardRelocatingAway() {
usagesBuilder.put("node2", new DiskUsage("node2", "n2", "/dev/null", 100, 100)); // 0% used
ImmutableOpenMap usages = usagesBuilder.build();
- ImmutableOpenMap.Builder shardSizesBuilder = ImmutableOpenMap.builder();
- shardSizesBuilder.put("[test][0][p]", 40L);
- shardSizesBuilder.put("[test][1][p]", 40L);
- shardSizesBuilder.put("[foo][0][p]", 10L);
- ImmutableOpenMap shardSizes = shardSizesBuilder.build();
+ ImmutableOpenMap.Builder shardSizesBuilder = ImmutableOpenMap.builder();
+ shardSizesBuilder.put(shardId("test", 0), 40L);
+ shardSizesBuilder.put(shardId("test", 1), 40L);
+ shardSizesBuilder.put(shardId("foo", 0), 10L);
+ ImmutableOpenMap shardSizes = shardSizesBuilder.build();
final ClusterInfo clusterInfo = new DevNullClusterInfo(usages, usages, shardSizes);
DiskThresholdDecider diskThresholdDecider = makeDecider(diskSettings);
MetaData metaData = MetaData.builder()
- .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(0))
- .put(IndexMetaData.builder("foo").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0))
+ .put(indexMetaData("test", 2, 0))
+ .put(indexMetaData("foo", 1, 0))
.build();
RoutingTable initialRoutingTable = RoutingTable.builder()
@@ -808,8 +806,8 @@ public void testCanRemainWithShardRelocatingAway() {
.build();
// Two shards consuming each 80% of disk space while 70% is allowed, so shard 0 isn't allowed here
- ShardRouting firstRouting = TestShardRouting.newShardRouting("test", 0, "node1", null, true, ShardRoutingState.STARTED);
- ShardRouting secondRouting = TestShardRouting.newShardRouting("test", 1, "node1", null, true, ShardRoutingState.STARTED);
+ ShardRouting firstRouting = TestShardRouting.newShardRouting(shardId("test", 0), "node1", null, true, ShardRoutingState.STARTED);
+ ShardRouting secondRouting = TestShardRouting.newShardRouting(shardId("test", 1), "node1", null, true, ShardRoutingState.STARTED);
RoutingNode firstRoutingNode = new RoutingNode("node1", discoveryNode1, firstRouting, secondRouting);
RoutingTable.Builder builder = RoutingTable.builder().add(
IndexRoutingTable.builder(firstRouting.index())
@@ -834,9 +832,9 @@ public void testCanRemainWithShardRelocatingAway() {
"actual free: [20.0%]"));
// Two shards consuming each 80% of disk space while 70% is allowed, but one is relocating, so shard 0 can stay
- firstRouting = TestShardRouting.newShardRouting("test", 0, "node1", null, true, ShardRoutingState.STARTED);
- secondRouting = TestShardRouting.newShardRouting("test", 1, "node1", "node2", true, ShardRoutingState.RELOCATING);
- ShardRouting fooRouting = TestShardRouting.newShardRouting("foo", 0, null, true, ShardRoutingState.UNASSIGNED);
+ firstRouting = TestShardRouting.newShardRouting(shardId("test", 0), "node1", null, true, ShardRoutingState.STARTED);
+ secondRouting = TestShardRouting.newShardRouting(shardId("test", 1), "node1", "node2", true, ShardRoutingState.RELOCATING);
+ ShardRouting fooRouting = TestShardRouting.newShardRouting(shardId("foo", 0), null, true, ShardRoutingState.UNASSIGNED);
firstRoutingNode = new RoutingNode("node1", discoveryNode1, firstRouting, secondRouting);
builder = RoutingTable.builder().add(
IndexRoutingTable.builder(firstRouting.index())
@@ -906,14 +904,14 @@ public void testForSingleDataNode() {
ImmutableOpenMap usages = usagesBuilder.build();
// We have an index with 1 primary shards each taking 40 bytes. Each node has 100 bytes available
- ImmutableOpenMap.Builder shardSizes = ImmutableOpenMap.builder();
- shardSizes.put("[test][0][p]", 40L);
- shardSizes.put("[test][1][p]", 40L);
+ ImmutableOpenMap.Builder shardSizes = ImmutableOpenMap.builder();
+ shardSizes.put(shardId("test", 0), 40L);
+ shardSizes.put(shardId("test", 1), 40L);
final ClusterInfo clusterInfo = new DevNullClusterInfo(usages, usages, shardSizes.build());
DiskThresholdDecider diskThresholdDecider = makeDecider(diskSettings);
MetaData metaData = MetaData.builder()
- .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(0))
+ .put(indexMetaData("test", 2, 0))
.build();
RoutingTable initialRoutingTable = RoutingTable.builder()
@@ -934,8 +932,8 @@ public void testForSingleDataNode() {
.build();
// Two shards consumes 80% of disk space in data node, but we have only one data node, shards should remain.
- ShardRouting firstRouting = TestShardRouting.newShardRouting("test", 0, "node2", null, true, ShardRoutingState.STARTED);
- ShardRouting secondRouting = TestShardRouting.newShardRouting("test", 1, "node2", null, true, ShardRoutingState.STARTED);
+ ShardRouting firstRouting = TestShardRouting.newShardRouting(shardId("test", 0), "node2", null, true, ShardRoutingState.STARTED);
+ ShardRouting secondRouting = TestShardRouting.newShardRouting(shardId("test", 1), "node2", null, true, ShardRoutingState.STARTED);
RoutingNode firstRoutingNode = new RoutingNode("node2", discoveryNode2, firstRouting, secondRouting);
RoutingTable.Builder builder = RoutingTable.builder().add(
@@ -988,8 +986,8 @@ Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLU
ClusterState updateClusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
.add(discoveryNode3)).build();
- firstRouting = TestShardRouting.newShardRouting("test", 0, "node2", null, true, ShardRoutingState.STARTED);
- secondRouting = TestShardRouting.newShardRouting("test", 1, "node2", "node3", true, ShardRoutingState.RELOCATING);
+ firstRouting = TestShardRouting.newShardRouting(shardId("test", 0), "node2", null, true, ShardRoutingState.STARTED);
+ secondRouting = TestShardRouting.newShardRouting(shardId("test", 1), "node2", "node3", true, ShardRoutingState.RELOCATING);
firstRoutingNode = new RoutingNode("node2", discoveryNode2, firstRouting, secondRouting);
builder = RoutingTable.builder().add(
IndexRoutingTable.builder(firstRouting.index())
@@ -1020,6 +1018,12 @@ Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLU
assertThat(result.routingTable().index("test").getShards().get(1).primaryShard().relocatingNodeId(), equalTo("node3"));
}
+ private IndexMetaData.Builder indexMetaData(String indexName, int numberOfShards, int numberOfReplicas) {
+ return IndexMetaData.builder(indexName).settings(settings(Version.CURRENT)
+ .put(IndexMetaData.SETTING_INDEX_UUID, shardId(indexName, 0).getIndex().getUUID()))
+ .numberOfShards(numberOfShards).numberOfReplicas(numberOfReplicas);
+ }
+
public void logShardStates(ClusterState state) {
RoutingNodes rn = state.getRoutingNodes();
logger.info("--> counts: total: {}, unassigned: {}, initializing: {}, relocating: {}, started: {}",
@@ -1041,7 +1045,7 @@ public void logShardStates(ClusterState state) {
static class DevNullClusterInfo extends ClusterInfo {
DevNullClusterInfo(ImmutableOpenMap leastAvailableSpaceUsage,
ImmutableOpenMap mostAvailableSpaceUsage,
- ImmutableOpenMap shardSizes) {
+ ImmutableOpenMap shardSizes) {
super(leastAvailableSpaceUsage, mostAvailableSpaceUsage, shardSizes, null);
}
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
index fe7bfd9dc804c..ebc31afe43792 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
@@ -101,8 +101,8 @@ public void testCanAllocateUsesMaxAvailableSpace() {
// this is weird and smells like a bug! it should be up to 20%?
mostAvailableUsage.put("node_1", new DiskUsage("node_1", "node_1", "_na_", 100, randomIntBetween(0, 10)));
- ImmutableOpenMap.Builder shardSizes = ImmutableOpenMap.builder();
- shardSizes.put("[test][0][p]", 10L); // 10 bytes
+ ImmutableOpenMap.Builder shardSizes = ImmutableOpenMap.builder();
+ shardSizes.put(new ShardId(index, 0), 10L); // 10 bytes
final ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages.build(),
mostAvailableUsage.build(), shardSizes.build(), ImmutableOpenMap.of());
RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Collections.singleton(decider)),
@@ -154,10 +154,10 @@ public void testCannotAllocateDueToLackOfDiskResources() {
final int freeBytes = randomIntBetween(20, 100);
mostAvailableUsage.put("node_0", new DiskUsage("node_0", "node_0", "_na_", 100, freeBytes));
- ImmutableOpenMap.Builder shardSizes = ImmutableOpenMap.builder();
+ ImmutableOpenMap.Builder shardSizes = ImmutableOpenMap.builder();
// way bigger than available space
final long shardSize = randomIntBetween(110, 1000);
- shardSizes.put("[test][0][p]", shardSize);
+ shardSizes.put(new ShardId(index, 0), shardSize);
ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages.build(), mostAvailableUsage.build(),
shardSizes.build(), ImmutableOpenMap.of());
RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Collections.singleton(decider)),
@@ -234,10 +234,10 @@ public void testCanRemainUsesLeastAvailableSpace() {
mostAvailableUsage.put("node_0", new DiskUsage("node_0", "node_0", "/node0/most", 100, 90)); // 10% used
mostAvailableUsage.put("node_1", new DiskUsage("node_1", "node_1", "/node1/most", 100, 90)); // 10% used
- ImmutableOpenMap.Builder shardSizes = ImmutableOpenMap.builder();
- shardSizes.put("[test][0][p]", 10L); // 10 bytes
- shardSizes.put("[test][1][p]", 10L);
- shardSizes.put("[test][2][p]", 10L);
+ ImmutableOpenMap.Builder shardSizes = ImmutableOpenMap.builder();
+ shardSizes.put(new ShardId(indexMetaData.getIndex(), 0), 10L); // 10 bytes
+ shardSizes.put(new ShardId(indexMetaData.getIndex(), 1), 10L);
+ shardSizes.put(new ShardId(indexMetaData.getIndex(), 2), 10L);
final ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages.build(), mostAvailableUsage.build(),
shardSizes.build(), shardRoutingMap.build());
@@ -279,12 +279,6 @@ public void testCanRemainUsesLeastAvailableSpace() {
public void testShardSizeAndRelocatingSize() {
- ImmutableOpenMap.Builder shardSizes = ImmutableOpenMap.builder();
- shardSizes.put("[test][0][r]", 10L);
- shardSizes.put("[test][1][r]", 100L);
- shardSizes.put("[test][2][r]", 1000L);
- shardSizes.put("[other][0][p]", 10000L);
- ClusterInfo info = new DevNullClusterInfo(ImmutableOpenMap.of(), ImmutableOpenMap.of(), shardSizes.build());
MetaData.Builder metaBuilder = MetaData.builder();
metaBuilder.put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)
.put("index.uuid", "1234")).numberOfShards(3).numberOfReplicas(1));
@@ -296,9 +290,17 @@ public void testShardSizeAndRelocatingSize() {
routingTableBuilder.addAsNew(metaData.index("other"));
ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING
.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTableBuilder.build()).build();
+
+ ImmutableOpenMap.Builder shardSizes = ImmutableOpenMap.builder();
+ shardSizes.put(new ShardId(metaData.index("test").getIndex(), 0), 10L); // 10 bytes
+ shardSizes.put(new ShardId(metaData.index("test").getIndex(), 1), 100L);
+ shardSizes.put(new ShardId(metaData.index("test").getIndex(), 2), 1000L);
+ shardSizes.put(new ShardId(metaData.index("other").getIndex(), 0), 10000L);
+ ClusterInfo info = new DevNullClusterInfo(ImmutableOpenMap.of(), ImmutableOpenMap.of(), shardSizes.build());
RoutingAllocation allocation = new RoutingAllocation(null, null, clusterState, info, 0);
- final Index index = new Index("test", "1234");
+
+ final Index index = metaData.index("test").getIndex();
ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), false, PeerRecoverySource.INSTANCE,
new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
test_0 = ShardRoutingHelper.initialize(test_0, "node1");
@@ -321,7 +323,7 @@ public void testShardSizeAndRelocatingSize() {
assertEquals(10L, getExpectedShardSize(test_0, 0L, allocation));
RoutingNode node = new RoutingNode("node1", new DiscoveryNode("node1", buildNewFakeTransportAddress(),
- emptyMap(), emptySet(), Version.CURRENT), test_0, test_1.getTargetRelocatingShard(), test_2);
+ emptyMap(), emptySet(), Version.CURRENT), test_0, test_1.getTargetRelocatingShard(), test_2);
assertEquals(100L, sizeOfRelocatingShards(allocation, node, false, "/dev/null"));
assertEquals(90L, sizeOfRelocatingShards(allocation, node, true, "/dev/null"));
assertEquals(0L, sizeOfRelocatingShards(allocation, node, true, "/dev/some/other/dev"));
@@ -334,7 +336,7 @@ public void testShardSizeAndRelocatingSize() {
assertEquals(0L, getExpectedShardSize(test_3, 0L, allocation));
- ShardRouting other_0 = ShardRouting.newUnassigned(new ShardId("other", "5678", 0), randomBoolean(),
+ ShardRouting other_0 = ShardRouting.newUnassigned(new ShardId(metaData.index("other").getIndex(), 0), randomBoolean(),
PeerRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
other_0 = ShardRoutingHelper.initialize(other_0, "node2");
other_0 = ShardRoutingHelper.moveToStarted(other_0);
@@ -342,13 +344,8 @@ public void testShardSizeAndRelocatingSize() {
node = new RoutingNode("node1", new DiscoveryNode("node1", buildNewFakeTransportAddress(), emptyMap(), emptySet(),
Version.CURRENT), test_0, test_1.getTargetRelocatingShard(), test_2, other_0.getTargetRelocatingShard());
- if (other_0.primary()) {
- assertEquals(10100L, sizeOfRelocatingShards(allocation, node, false, "/dev/null"));
- assertEquals(10090L, sizeOfRelocatingShards(allocation, node, true, "/dev/null"));
- } else {
- assertEquals(100L, sizeOfRelocatingShards(allocation, node, false, "/dev/null"));
- assertEquals(90L, sizeOfRelocatingShards(allocation, node, true, "/dev/null"));
- }
+ assertEquals(10100L, sizeOfRelocatingShards(allocation, node, false, "/dev/null"));
+ assertEquals(10090L, sizeOfRelocatingShards(allocation, node, true, "/dev/null"));
}
public long sizeOfRelocatingShards(RoutingAllocation allocation, RoutingNode node, boolean subtractShardsMovingAway, String dataPath) {
@@ -357,13 +354,6 @@ public long sizeOfRelocatingShards(RoutingAllocation allocation, RoutingNode nod
}
public void testSizeShrinkIndex() {
- ImmutableOpenMap.Builder shardSizes = ImmutableOpenMap.builder();
- shardSizes.put("[test][0][p]", 10L);
- shardSizes.put("[test][1][p]", 100L);
- shardSizes.put("[test][2][p]", 500L);
- shardSizes.put("[test][3][p]", 500L);
-
- ClusterInfo info = new DevNullClusterInfo(ImmutableOpenMap.of(), ImmutableOpenMap.of(), shardSizes.build());
MetaData.Builder metaBuilder = MetaData.builder();
metaBuilder.put(IndexMetaData.builder("test").settings(settings(Version.CURRENT).put("index.uuid", "1234"))
.numberOfShards(4).numberOfReplicas(0));
@@ -382,6 +372,13 @@ public void testSizeShrinkIndex() {
ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING
.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTableBuilder.build()).build();
+ ImmutableOpenMap.Builder shardSizes = ImmutableOpenMap.builder();
+ shardSizes.put(new ShardId(metaData.index("test").getIndex(), 0), 10L); // 10 bytes
+ shardSizes.put(new ShardId(metaData.index("test").getIndex(), 1), 100L);
+ shardSizes.put(new ShardId(metaData.index("test").getIndex(), 2), 500L);
+ shardSizes.put(new ShardId(metaData.index("test").getIndex(), 3), 500L);
+ ClusterInfo info = new DevNullClusterInfo(ImmutableOpenMap.of(), ImmutableOpenMap.of(), shardSizes.build());
+
AllocationService allocationService = createAllocationService();
clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")))
.build();
@@ -392,7 +389,7 @@ public void testSizeShrinkIndex() {
RoutingAllocation allocation = new RoutingAllocation(null, clusterState.getRoutingNodes(), clusterState, info, 0);
- final Index index = new Index("test", "1234");
+ final Index index = metaData.index("test").getIndex();
ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), true,
LocalShardsRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
test_0 = ShardRoutingHelper.initialize(test_0, "node1");