From 76474dceb9d194c3a41e4823c4f4a9ec7dc1436b Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Mon, 29 Oct 2018 19:13:21 +0300 Subject: [PATCH 01/57] [Zen2] Write manifest file --- .../cluster/metadata/IndexMetaData.java | 5 + .../cluster/metadata/MetaData.java | 5 + .../cluster/metadata/MetaState.java | 188 ++++++++ .../gateway/GatewayMetaState.java | 422 ++++++++++++------ .../gateway/IndexMetaDataWriter.java | 28 ++ .../gateway/MetaDataStateFormat.java | 96 ++-- .../gateway/MetaStateService.java | 193 ++++++-- .../TransportNodesListGatewayMetaState.java | 2 +- .../gateway/WriteStateException.java | 6 +- .../java/org/elasticsearch/node/Node.java | 2 +- .../admin/indices/create/CreateIndexIT.java | 6 +- .../gateway/GatewayIndexStateIT.java | 13 +- .../gateway/GatewayMetaStateTests.java | 358 ++++++++------- .../gateway/MetaDataStateFormatTests.java | 7 +- .../gateway/MetaDataWriteDataNodesIT.java | 2 +- .../gateway/MetaStateServiceTests.java | 94 +++- .../elasticsearch/gateway/MetaStateTests.java | 89 ++++ .../indices/IndicesServiceTests.java | 4 +- 18 files changed, 1150 insertions(+), 370 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/cluster/metadata/MetaState.java create mode 100644 server/src/main/java/org/elasticsearch/gateway/IndexMetaDataWriter.java create mode 100644 server/src/test/java/org/elasticsearch/gateway/MetaStateTests.java diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index b4a7733c62eed..ebfdf89f4a412 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -1391,6 +1391,11 @@ public static Settings addHumanReadableSettings(Settings settings) { */ public static final MetaDataStateFormat FORMAT = new MetaDataStateFormat(INDEX_STATE_FILE_PREFIX) { + @Override + protected boolean autoCleanup() { + return false; + } + @Override public void toXContent(XContentBuilder builder, IndexMetaData state) throws IOException { Builder.toXContent(state, builder, FORMAT_PARAMS); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index f4eafd05e1599..35e0ef3076cee 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -1297,6 +1297,11 @@ public static MetaData fromXContent(XContentParser parser) throws IOException { */ public static final MetaDataStateFormat FORMAT = new MetaDataStateFormat(GLOBAL_STATE_FILE_PREFIX) { + @Override + protected boolean autoCleanup() { + return false; + } + @Override public void toXContent(XContentBuilder builder, MetaData state) throws IOException { Builder.toXContent(state, builder, FORMAT_PARAMS); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaState.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaState.java new file mode 100644 index 0000000000000..98d2a2dfa7116 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaState.java @@ -0,0 +1,188 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster.metadata; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.gateway.MetaDataStateFormat; +import org.elasticsearch.index.Index; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * This class represents the manifest file, which is the entry point for reading meta data from disk. + * Metadata consists of global metadata and index metadata. + * When new version of metadata is written it's assigned some generation long value. + * Global metadata generation could be obtained by calling {@link #getGlobalStateGeneration()}. + * Index metadata generation could be obtained by calling {@link #getIndices()}. + */ +public class MetaState implements ToXContentFragment { + private final long globalStateGeneration; + private final Map indices; + + public MetaState(long globalStateGeneration, Map indices) { + this.globalStateGeneration = globalStateGeneration; + this.indices = indices; + } + + /** + * Returns global metadata generation. + */ + public long getGlobalStateGeneration() { + return globalStateGeneration; + } + + /** + * Returns map from {@link Index} to index metadata generation. + */ + public Map getIndices() { + return indices; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + MetaState metaState = (MetaState) o; + return globalStateGeneration == metaState.globalStateGeneration && + Objects.equals(indices, metaState.indices); + } + + @Override + public int hashCode() { + return Objects.hash(globalStateGeneration, indices); + } + + private static final String META_STATE_FILE_PREFIX = "meta-"; + private static final ToXContent.Params METASTATE_FORMAT_PARAMS = new ToXContent.MapParams(Collections.singletonMap("binary", "true")); + + public static final MetaDataStateFormat FORMAT = new MetaDataStateFormat(META_STATE_FILE_PREFIX) { + + @Override + protected boolean autoCleanup() { + return false; + } + + @Override + public void toXContent(XContentBuilder builder, MetaState state) throws IOException { + state.toXContent(builder, METASTATE_FORMAT_PARAMS); + } + + @Override + public MetaState fromXContent(XContentParser parser) throws IOException { + return MetaState.fromXContent(parser); + } + }; + + + /* + * Code below this comment is for XContent manipulation + */ + + private static final ParseField INDICES_PARSE_FIELD = new ParseField("indices"); + private static final ParseField GENERATION_PARSE_FIELD = new ParseField("generation"); + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(GENERATION_PARSE_FIELD.getPreferredName(), globalStateGeneration); + builder.array(INDICES_PARSE_FIELD.getPreferredName(), indexEntryList().toArray()); + return builder; + } + + private List indexEntryList() { + return indices.entrySet().stream(). + map(entry -> new IndexEntry(entry.getKey(), entry.getValue())). + collect(Collectors.toList()); + } + + private static long generation(Object[] generationAndListOfIndexEntries) { + return (Long) generationAndListOfIndexEntries[0]; + } + + private static Map indicies(Object[] generationAndListOfIndexEntries) { + List listOfIndices = (List) generationAndListOfIndexEntries[1]; + return listOfIndices.stream().collect(Collectors.toMap(IndexEntry::getIndex, IndexEntry::getGeneration)); + } + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "state", + generationAndListOfIndexEntries -> + new MetaState(generation(generationAndListOfIndexEntries), indicies(generationAndListOfIndexEntries))); + + static { + PARSER.declareLong(ConstructingObjectParser.constructorArg(), GENERATION_PARSE_FIELD); + PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), IndexEntry.INDEX_ENTRY_PARSER, INDICES_PARSE_FIELD); + } + + public static MetaState fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + private static final class IndexEntry implements ToXContentFragment { + private static final ParseField INDEX_GENERATION_PARSE_FIELD = new ParseField("generation"); + private static final ParseField INDEX_PARSE_FIELD = new ParseField("index"); + + static final ConstructingObjectParser INDEX_ENTRY_PARSER = new ConstructingObjectParser<>( + "indexEntry", + indexAndGeneration -> new IndexEntry((Index) indexAndGeneration[0], (long) indexAndGeneration[1])); + + static { + INDEX_ENTRY_PARSER.declareField(ConstructingObjectParser.constructorArg(), + Index::fromXContent, INDEX_PARSE_FIELD, ObjectParser.ValueType.OBJECT); + INDEX_ENTRY_PARSER.declareLong(ConstructingObjectParser.constructorArg(), INDEX_GENERATION_PARSE_FIELD); + } + + private final long generation; + private final Index index; + + IndexEntry(Index index, long generation) { + this.index = index; + this.generation = generation; + } + + public long getGeneration() { + return generation; + } + + public Index getIndex() { + return index; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(INDEX_PARSE_FIELD.getPreferredName(), index); + builder.field(GENERATION_PARSE_FIELD.getPreferredName(), generation); + builder.endObject(); + return builder; + } + } +} + diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index 46ff2f960e7cf..40741a6a1a3d5 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -27,11 +27,13 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService; +import org.elasticsearch.cluster.metadata.MetaState; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; @@ -54,18 +56,26 @@ import java.util.function.Consumer; import java.util.function.UnaryOperator; -import static java.util.Collections.emptySet; -import static java.util.Collections.unmodifiableSet; - +/** + * This class is responsible for storing/retrieving metadata to/from disk. + * When instance of this class is created, constructor ensures that this version is compatible with state stored on disk and performs + * state upgrade if necessary. Also it checks that atomic move is supported on the filesystem level, because it's a must for metadata + * store algorithm. + * Please note that the state being loaded when constructing the instance of this class is NOT the state that will be used as a + * {@link ClusterState#metaData()}. Instead when node is starting up, it calls {@link #loadMetaData()} method and if this node is + * elected as master, it requests metaData from other master eligible nodes. After that, master node performs re-conciliation on the + * gathered results, re-creates {@link ClusterState} and broadcasts this state to other nodes in the cluster. + * It means that the first time {@link #applyClusterState(ClusterChangedEvent)} method is called, it won't have any previous metaData in + * memory and will iterate over all the indices in received {@link ClusterState} and store them to disk. + */ public class GatewayMetaState extends AbstractComponent implements ClusterStateApplier { private final NodeEnvironment nodeEnv; private final MetaStateService metaStateService; @Nullable - private volatile MetaData previousMetaData; - - private volatile Set previouslyWrittenIndices = emptySet(); + //there is happens-before order between subsequent applyClusterState calls, hence no volatile modifier + private Tuple ourState; public GatewayMetaState(Settings settings, NodeEnvironment nodeEnv, MetaStateService metaStateService, MetaDataIndexUpgradeService metaDataIndexUpgradeService, MetaDataUpgrader metaDataUpgrader) throws IOException { @@ -73,34 +83,65 @@ public GatewayMetaState(Settings settings, NodeEnvironment nodeEnv, MetaStateSer this.nodeEnv = nodeEnv; this.metaStateService = metaStateService; - if (DiscoveryNode.isDataNode(settings)) { - ensureNoPre019ShardState(nodeEnv); - } + ensureNoPre019State(); + ensureAtomicMoveSupported(); + maybeUpgradeMetaData(metaDataIndexUpgradeService, metaDataUpgrader); + profileLoadMetaData(); + } + private void profileLoadMetaData() throws IOException { if (DiscoveryNode.isMasterNode(settings) || DiscoveryNode.isDataNode(settings)) { - nodeEnv.ensureAtomicMoveSupported(); + long startNS = System.nanoTime(); + metaStateService.loadFullState(); + logger.debug("took {} to load state", TimeValue.timeValueMillis(TimeValue.nsecToMSec(System.nanoTime() - startNS))); } + } + + private void maybeUpgradeMetaData(MetaDataIndexUpgradeService metaDataIndexUpgradeService, MetaDataUpgrader metaDataUpgrader) + throws IOException { if (DiscoveryNode.isMasterNode(settings) || DiscoveryNode.isDataNode(settings)) { try { - ensureNoPre019State(); - final MetaData metaData = metaStateService.loadFullState(); - final MetaData upgradedMetaData = upgradeMetaData(metaData, metaDataIndexUpgradeService, metaDataUpgrader); + final Tuple metaStateAndData = metaStateService.loadFullState(); + final MetaState metaState = metaStateAndData.v1(); + final MetaData metaData = metaStateAndData.v2(); + // We finished global state validation and successfully checked all indices for backward compatibility // and found no non-upgradable indices, which means the upgrade can continue. // Now it's safe to overwrite global and index metadata. - if (metaData != upgradedMetaData) { + long globalStateGeneration = metaState.getGlobalStateGeneration(); + + if (globalStateGeneration != -1) { + List cleanupActions = new ArrayList<>(); + // If globalStateGeneration is non-negative, it means we should have some metadata on disk + // Always re-write it even if upgrade plugins do not upgrade it, to be sure it's properly persisted on all data path + Map indices = new HashMap<>(metaState.getIndices()); + final MetaData upgradedMetaData = upgradeMetaData(metaData, metaDataIndexUpgradeService, metaDataUpgrader); + if (MetaData.isGlobalStateEquals(metaData, upgradedMetaData) == false) { - metaStateService.writeGlobalState("upgrade", upgradedMetaData); + globalStateGeneration = metaStateService.writeGlobalState("upgrade", upgradedMetaData); + } else { + globalStateGeneration = metaStateService.writeGlobalState("startup", metaData); } + final long currentGlobalStateGeneration = globalStateGeneration; + cleanupActions.add(() -> metaStateService.cleanupGlobalState(currentGlobalStateGeneration)); + for (IndexMetaData indexMetaData : upgradedMetaData) { + long generation; if (metaData.hasIndexMetaData(indexMetaData) == false) { - metaStateService.writeIndex("upgrade", indexMetaData); + generation = metaStateService.writeIndex("upgrade", indexMetaData); + } else { + generation = metaStateService.writeIndex("startup", indexMetaData); } + final long currentGeneration = generation; + cleanupActions.add(() -> metaStateService.cleanupIndex(indexMetaData.getIndex(), currentGeneration)); + indices.put(indexMetaData.getIndex(), generation); } + + final long metaStateGeneration = + metaStateService.writeMetaState("startup", new MetaState(globalStateGeneration, indices)); + cleanupActions.add(()->metaStateService.cleanupMetaState(metaStateGeneration)); + performCleanup(cleanupActions); } - long startNS = System.nanoTime(); - metaStateService.loadFullState(); - logger.debug("took {} to load state", TimeValue.timeValueMillis(TimeValue.nsecToMSec(System.nanoTime() - startNS))); } catch (Exception e) { logger.error("failed to read local state, exiting...", e); throw e; @@ -108,79 +149,105 @@ public GatewayMetaState(Settings settings, NodeEnvironment nodeEnv, MetaStateSer } } - public MetaData loadMetaState() throws IOException { - return metaStateService.loadFullState(); + private void ensureAtomicMoveSupported() throws IOException { + if (DiscoveryNode.isMasterNode(settings) || DiscoveryNode.isDataNode(settings)) { + nodeEnv.ensureAtomicMoveSupported(); + } + } + + public MetaData loadMetaData() throws IOException { + return metaStateService.loadFullState().v2(); } @Override public void applyClusterState(ClusterChangedEvent event) { + ClusterState newState = event.state(); + if (newState.nodes().getLocalNode().isMasterNode() == false && newState.nodes().getLocalNode().isDataNode() == false) { + return; + } - final ClusterState state = event.state(); - if (state.blocks().disableStatePersistence()) { - // reset the current metadata, we need to start fresh... - this.previousMetaData = null; - previouslyWrittenIndices = emptySet(); + if (event.state().blocks().disableStatePersistence()) { + // reset the current state, we need to start fresh... + ourState = null; return; } - MetaData newMetaData = state.metaData(); - // we don't check if metaData changed, since we might be called several times and we need to check dangling... - Set relevantIndices = Collections.emptySet(); - boolean success = true; - // write the state if this node is a master eligible node or if it is a data node and has shards allocated on it - if (state.nodes().getLocalNode().isMasterNode() || state.nodes().getLocalNode().isDataNode()) { - if (previousMetaData == null) { - try { - // we determine if or if not we write meta data on data only nodes by looking at the shard routing - // and only write if a shard of this index is allocated on this node - // however, closed indices do not appear in the shard routing. if the meta data for a closed index is - // updated it will therefore not be written in case the list of previouslyWrittenIndices is empty (because state - // persistence was disabled or the node was restarted), see getRelevantIndicesOnDataOnlyNode(). - // we therefore have to check here if we have shards on disk and add their indices to the previouslyWrittenIndices list - if (isDataOnlyNode(state)) { - Set newPreviouslyWrittenIndices = new HashSet<>(previouslyWrittenIndices.size()); - for (IndexMetaData indexMetaData : newMetaData) { - IndexMetaData indexMetaDataOnDisk = null; - if (indexMetaData.getState().equals(IndexMetaData.State.CLOSE)) { - indexMetaDataOnDisk = metaStateService.loadIndexState(indexMetaData.getIndex()); - } - if (indexMetaDataOnDisk != null) { - newPreviouslyWrittenIndices.add(indexMetaDataOnDisk.getIndex()); - } - } - newPreviouslyWrittenIndices.addAll(previouslyWrittenIndices); - previouslyWrittenIndices = unmodifiableSet(newPreviouslyWrittenIndices); - } - } catch (Exception e) { - success = false; - } + try { + if (ourState == null) { + ourState = metaStateService.loadFullState(); } - // check if the global state changed? - if (previousMetaData == null || !MetaData.isGlobalStateEquals(previousMetaData, newMetaData)) { - try { - metaStateService.writeGlobalState("changed", newMetaData); - } catch (Exception e) { - success = false; - } + ourState = updateMetaData(event); + } catch (WriteStateException e) { + if (e.isDirty()) { + logger.error("Fatal exception occurred when storing new meta data. Storage is dirty", e); + } else { + logger.warn("Exception occurred when storing new meta data. Storage is not dirty", e); } + } catch (Exception e) { + logger.warn("Exception occurred when storing new meta data", e); + } + } + /** + * Updates meta state and meta data on disk according to {@link ClusterChangedEvent}. + * + * @throws IOException if IOException occurs. It's recommended for the callers of this method to handle {@link WriteStateException}, + * which is subclass of {@link IOException} explicitly. See also {@link WriteStateException#isDirty()}. + */ + private Tuple updateMetaData(ClusterChangedEvent event) throws IOException { + ClusterState newState = event.state(); + ClusterState previousState = event.previousState(); + MetaData newMetaData = newState.metaData(); + + List cleanupActions = new ArrayList<>(); + long globalStateGeneration = writeGlobalState(newMetaData, cleanupActions); + Map newIndices = writeIndicesMetadata(newState, previousState, cleanupActions); + MetaState metaState = new MetaState(globalStateGeneration, newIndices); + writeMetaState(metaState, cleanupActions); + performCleanup(cleanupActions); + return new Tuple<>(metaState, newMetaData); + } - relevantIndices = getRelevantIndices(event.state(), event.previousState(), previouslyWrittenIndices); - final Iterable writeInfo = resolveStatesToBeWritten(previouslyWrittenIndices, relevantIndices, previousMetaData, event.state().metaData()); - // check and write changes in indices - for (IndexMetaWriteInfo indexMetaWrite : writeInfo) { - try { - metaStateService.writeIndex(indexMetaWrite.reason, indexMetaWrite.newMetaData); - } catch (Exception e) { - success = false; - } - } + private void performCleanup(List cleanupActions) { + for (Runnable action : cleanupActions) { + action.run(); + } + } + + private void writeMetaState(MetaState metaState, List cleanupActions) throws IOException { + if (metaState.equals(ourState.v1()) == false) { + long generation = metaStateService.writeMetaState("changed", metaState); + cleanupActions.add(() -> metaStateService.cleanupMetaState(generation)); + } + } + + private Map writeIndicesMetadata(ClusterState newState, ClusterState previousState, List cleanupActions) + throws IOException { + MetaData previousMetadata = ourState.v2(); + MetaState previousMetastate = ourState.v1(); + Map previouslyWrittenIndices = previousMetastate.getIndices(); + Set relevantIndices = getRelevantIndices(newState, previousState, previouslyWrittenIndices.keySet()); + + Map newIndices = new HashMap<>(); + + Iterable actions = resolveIndexMetaDataActions(previouslyWrittenIndices, relevantIndices, previousMetadata, + newState.metaData()); + + for (IndexMetaDataAction action : actions) { + long generation = action.execute(metaStateService, cleanupActions); + newIndices.put(action.getIndex(), generation); } - if (success) { - previousMetaData = newMetaData; - previouslyWrittenIndices = unmodifiableSet(relevantIndices); + return newIndices; + } + + private long writeGlobalState(MetaData newMetaData, List cleanupActions) throws IOException { + if (ourState.v1().getGlobalStateGeneration() == -1 || MetaData.isGlobalStateEquals(ourState.v2(), newMetaData) == false) { + long generation = metaStateService.writeGlobalState("changed", newMetaData); + cleanupActions.add(() -> metaStateService.cleanupGlobalState(generation)); + return generation; } + return ourState.v1().getGlobalStateGeneration(); } public static Set getRelevantIndices(ClusterState state, ClusterState previousState, Set previouslyWrittenIndices) { @@ -196,14 +263,24 @@ public static Set getRelevantIndices(ClusterState state, ClusterState pre } - protected static boolean isDataOnlyNode(ClusterState state) { + private static boolean isDataOnlyNode(ClusterState state) { return ((state.nodes().getLocalNode().isMasterNode() == false) && state.nodes().getLocalNode().isDataNode()); } + + private void ensureNoPre019State() throws IOException { + if (DiscoveryNode.isDataNode(settings)) { + ensureNoPre019ShardState(); + } + if (DiscoveryNode.isMasterNode(settings) || DiscoveryNode.isDataNode(settings)) { + ensureNoPre019MetadataFiles(); + } + } + /** * Throws an IAE if a pre 0.19 state is detected */ - private void ensureNoPre019State() throws IOException { + private void ensureNoPre019MetadataFiles() throws IOException { for (Path dataLocation : nodeEnv.nodeDataPaths()) { final Path stateLocation = dataLocation.resolve(MetaDataStateFormat.STATE_DIR_NAME); if (!Files.exists(stateLocation)) { @@ -225,6 +302,22 @@ private void ensureNoPre019State() throws IOException { } } + // shard state BWC + private void ensureNoPre019ShardState() throws IOException { + for (Path dataLocation : nodeEnv.nodeDataPaths()) { + final Path stateLocation = dataLocation.resolve(MetaDataStateFormat.STATE_DIR_NAME); + if (Files.exists(stateLocation)) { + try (DirectoryStream stream = Files.newDirectoryStream(stateLocation, "shards-*")) { + for (Path stateFile : stream) { + throw new IllegalStateException("Detected pre 0.19 shard state file please upgrade to a version before " + + Version.CURRENT.minimumIndexCompatibilityVersion() + + " first to upgrade state structures - shard state found: [" + stateFile.getParent().toAbsolutePath()); + } + } + } + } + } + /** * Elasticsearch 2.0 removed several deprecated features and as well as support for Lucene 3.x. This method calls * {@link MetaDataIndexUpgradeService} to makes sure that indices are compatible with the current version. The @@ -235,7 +328,7 @@ private void ensureNoPre019State() throws IOException { */ static MetaData upgradeMetaData(MetaData metaData, MetaDataIndexUpgradeService metaDataIndexUpgradeService, - MetaDataUpgrader metaDataUpgrader) throws IOException { + MetaDataUpgrader metaDataUpgrader) { // upgrade index meta data boolean changed = false; final MetaData.Builder upgradedMetaData = MetaData.builder(metaData); @@ -280,52 +373,53 @@ private static boolean applyPluginUpgraders(ImmutableOpenMap stream = Files.newDirectoryStream(stateLocation, "shards-*")) { - for (Path stateFile : stream) { - throw new IllegalStateException("Detected pre 0.19 shard state file please upgrade to a version before " - + Version.CURRENT.minimumIndexCompatibilityVersion() - + " first to upgrade state structures - shard state found: [" + stateFile.getParent().toAbsolutePath()); - } - } - } - } - } - /** - * Loads the current meta state for each index in the new cluster state and checks if it has to be persisted. - * Each index state that should be written to disk will be returned. This is only run for data only nodes. - * It will return only the states for indices that actually have a shard allocated on the current node. + * Returns list of {@link IndexMetaDataAction} for each relevant index. + * For each relevant index there are 3 options: + *
    + *
  1. + * {@link KeepPreviousGeneration} - index metadata is already stored to disk and index metadata version is not changed, no + * action is required. + *
    2. + *
  2. + * {@link WriteNewIndexMetaData} - there is no index metadata on disk and index metadata for this index should be written. + *
    3. + *
  3. + * {@link WriteChangedIndexMetaData} - index metadata is already on disk, but index metadata version has changed. Updated + * index metadata should be written to disk. + *
    4. + *
* - * @param previouslyWrittenIndices A list of indices for which the state was already written before - * @param potentiallyUnwrittenIndices The list of indices for which state should potentially be written - * @param previousMetaData The last meta data we know of. meta data for all indices in previouslyWrittenIndices list is persisted now - * @param newMetaData The new metadata - * @return iterable over all indices states that should be written to disk + * @param previouslyWrittenIndices A list of indices for which the state was already written before + * @param relevantIndices The list of indices for which state should potentially be written + * @param previousMetaData The last meta data we know of + * @param newMetaData The new metadata + * @return list of {@link IndexMetaDataAction} for each relevant index. */ - public static Iterable resolveStatesToBeWritten(Set previouslyWrittenIndices, Set potentiallyUnwrittenIndices, MetaData previousMetaData, MetaData newMetaData) { - List indicesToWrite = new ArrayList<>(); - for (Index index : potentiallyUnwrittenIndices) { - IndexMetaData newIndexMetaData = newMetaData.getIndexSafe(index); - IndexMetaData previousIndexMetaData = previousMetaData == null ? null : previousMetaData.index(index); - String writeReason = null; - if (previouslyWrittenIndices.contains(index) == false || previousIndexMetaData == null) { - writeReason = "freshly created"; - } else if (previousIndexMetaData.getVersion() != newIndexMetaData.getVersion()) { - writeReason = "version changed from [" + previousIndexMetaData.getVersion() + "] to [" + newIndexMetaData.getVersion() + "]"; - } - if (writeReason != null) { - indicesToWrite.add(new GatewayMetaState.IndexMetaWriteInfo(newIndexMetaData, previousIndexMetaData, writeReason)); + public static List resolveIndexMetaDataActions(Map previouslyWrittenIndices, + Set relevantIndices, + MetaData previousMetaData, + MetaData newMetaData) { + List actions = new ArrayList<>(); + for (Index index : relevantIndices) { + Long generation = previouslyWrittenIndices.get(index); + IndexMetaData newIndexMetadata = newMetaData.getIndexSafe(index); + if (generation == null) { + actions.add(new WriteNewIndexMetaData(newIndexMetadata)); + } else { + IndexMetaData previousIndexMetadata = previousMetaData.index(index); + if (previousIndexMetadata.getVersion() != newIndexMetadata.getVersion()) { + actions.add(new WriteChangedIndexMetaData(previousIndexMetadata, newIndexMetadata)); + } else { + actions.add(new KeepPreviousGeneration(index, generation)); + } } } - return indicesToWrite; + return actions; } - public static Set getRelevantIndicesOnDataOnlyNode(ClusterState state, ClusterState previousState, Set previouslyWrittenIndices) { + private static Set getRelevantIndicesOnDataOnlyNode(ClusterState state, ClusterState previousState, Set + previouslyWrittenIndices) { RoutingNode newRoutingNode = state.getRoutingNodes().node(state.nodes().getLocalNodeId()); if (newRoutingNode == null) { throw new IllegalStateException("cluster state does not contain this node - cannot write index meta state"); @@ -350,7 +444,7 @@ public static Set getRelevantIndicesOnDataOnlyNode(ClusterState state, Cl return indices; } - public static Set getRelevantIndicesForMasterEligibleNode(ClusterState state) { + private static Set getRelevantIndicesForMasterEligibleNode(ClusterState state) { Set relevantIndices; relevantIndices = new HashSet<>(); // we have to iterate over the metadata to make sure we also capture closed indices @@ -360,24 +454,88 @@ public static Set getRelevantIndicesForMasterEligibleNode(ClusterState st return relevantIndices; } + /** + * Action to perform with index metadata. + */ + public interface IndexMetaDataAction { + /** + * @return index for index metadata. + */ + Index getIndex(); + + /** + * Executes this action using writer and potentially adding cleanup action to the list of cleanupActions. + * + * @param writer entity that can write index metadata to disk and perform cleanup afterwards. We prefer + * {@link IndexMetaDataWriter} interface in place of {@link MetaStateService} for easier unit testing. + * @param cleanupActions list of actions, which is expected to be mutated by adding new clean up action to it. + * @return new index metadata state generation, to be used in manifest file. + * @throws WriteStateException if exception occurs. + */ + long execute(IndexMetaDataWriter writer, List cleanupActions) throws WriteStateException; + } + + public static class KeepPreviousGeneration implements IndexMetaDataAction { + private final Index index; + private final long generation; + + KeepPreviousGeneration(Index index, long generation) { + this.index = index; + this.generation = generation; + } + + @Override + public Index getIndex() { + return index; + } + + @Override + public long execute(IndexMetaDataWriter writer, List cleanupActions) { + return generation; + } + } + + public static class WriteNewIndexMetaData implements IndexMetaDataAction { + private final IndexMetaData indexMetaData; + + WriteNewIndexMetaData(IndexMetaData indexMetaData) { + this.indexMetaData = indexMetaData; + } + + @Override + public Index getIndex() { + return indexMetaData.getIndex(); + } + + @Override + public long execute(IndexMetaDataWriter writer, List cleanupActions) throws WriteStateException { + long generation = writer.writeIndex("freshly created", indexMetaData); + cleanupActions.add(() -> writer.cleanupIndex(indexMetaData.getIndex(), generation)); + return generation; + } + } - public static class IndexMetaWriteInfo { - final IndexMetaData newMetaData; - final String reason; - final IndexMetaData previousMetaData; + public static class WriteChangedIndexMetaData implements IndexMetaDataAction { + private final IndexMetaData newIndexMetaData; + private final IndexMetaData oldIndexMetaData; - public IndexMetaWriteInfo(IndexMetaData newMetaData, IndexMetaData previousMetaData, String reason) { - this.newMetaData = newMetaData; - this.reason = reason; - this.previousMetaData = previousMetaData; + WriteChangedIndexMetaData(IndexMetaData oldIndexMetaData, IndexMetaData newIndexMetaData) { + this.oldIndexMetaData = oldIndexMetaData; + this.newIndexMetaData = newIndexMetaData; } - public IndexMetaData getNewMetaData() { - return newMetaData; + @Override + public Index getIndex() { + return newIndexMetaData.getIndex(); } - public String getReason() { - return reason; + @Override + public long execute(IndexMetaDataWriter writer, List cleanupActions) throws WriteStateException { + long generation = writer.writeIndex( + "version changed from [" + oldIndexMetaData.getVersion() + "] to [" + newIndexMetaData.getVersion() + "]", + newIndexMetaData); + cleanupActions.add(() -> writer.cleanupIndex(newIndexMetaData.getIndex(), generation)); + return generation; } } } diff --git a/server/src/main/java/org/elasticsearch/gateway/IndexMetaDataWriter.java b/server/src/main/java/org/elasticsearch/gateway/IndexMetaDataWriter.java new file mode 100644 index 0000000000000..45d6235ef966d --- /dev/null +++ b/server/src/main/java/org/elasticsearch/gateway/IndexMetaDataWriter.java @@ -0,0 +1,28 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gateway; + +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.index.Index; + +public interface IndexMetaDataWriter { + long writeIndex(String reason, IndexMetaData indexMetaData) throws WriteStateException; + void cleanupIndex(Index index, long currentGeneration); +} diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java b/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java index 7ae6a13724d66..775d7b18996e6 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java @@ -114,7 +114,6 @@ public void close() { // in order to write the footer we need to prevent closing the actual index input. } })) { - builder.startObject(); toXContent(builder, state); builder.endObject(); @@ -183,14 +182,18 @@ private static void performStateDirectoriesFsync(List> st * doesn't exist. The state is serialized to a temporary file in that directory and is then atomically moved to * it's target filename of the pattern {@code {prefix}{version}.st}. * If this method returns without exception there is a guarantee that state is persisted to the disk and loadLatestState will return - * it. + * it.
+ * If {@link #autoCleanup()} returns false, this method does not perform cleanup of old state files, + * because one write could be a part of larger transaction. + * If this write succeeds, but some further write fails, you may want to rollback the transaction and keep old file around. + * After transaction is finished use {@link #cleanupOldFiles(long, Path[])} for the clean-up. * * @param state the state object to write * @param locations the locations where the state should be written to. * @throws WriteStateException if some exception during writing state occurs. See also {@link WriteStateException#isDirty()}. + * @return generation of newly written state. */ - - public final void write(final T state, final Path... locations) throws WriteStateException { + public final long write(final T state, final Path... locations) throws WriteStateException { if (locations == null) { throw new IllegalArgumentException("Locations must not be null"); } @@ -231,7 +234,11 @@ public final void write(final T state, final Path... locations) throws WriteStat } } - cleanupOldFiles(fileName, locations); + if (autoCleanup()) { + cleanupOldFiles(maxStateId, locations); + } + + return maxStateId; } protected XContentBuilder newXContentBuilder(XContentType type, OutputStream stream ) throws IOException { @@ -257,7 +264,7 @@ protected XContentBuilder newXContentBuilder(XContentType type, OutputStream str public final T read(NamedXContentRegistry namedXContentRegistry, Path file) throws IOException { try (Directory dir = newDirectory(file.getParent())) { try (IndexInput indexInput = dir.openInput(file.getFileName().toString(), IOContext.DEFAULT)) { - // We checksum the entire file before we even go and parse it. If it's corrupted we barf right here. + // We checksum the entire file before we even go and parse it. If it's corrupted we barf right here. CodecUtil.checksumEntireFile(indexInput); CodecUtil.checkHeader(indexInput, STATE_FILE_CODEC, MIN_COMPATIBLE_STATE_FILE_VERSION, STATE_FILE_VERSION); final XContentType xContentType = XContentType.values()[indexInput.readInt()]; @@ -269,7 +276,7 @@ public final T read(NamedXContentRegistry namedXContentRegistry, Path file) thro try (IndexInput slice = indexInput.slice("state_xcontent", filePointer, contentSize)) { try (XContentParser parser = XContentFactory.xContent(FORMAT) .createParser(namedXContentRegistry, LoggingDeprecationHandler.INSTANCE, - new InputStreamIndexInput(slice, contentSize))) { + new InputStreamIndexInput(slice, contentSize))) { return fromXContent(parser); } } @@ -284,13 +291,24 @@ protected Directory newDirectory(Path dir) throws IOException { return new SimpleFSDirectory(dir); } - private void cleanupOldFiles(final String currentStateFile, Path[] locations) { + protected boolean autoCleanup() { + return true; + } + + /** + * Clean ups all state files not matching passed generation. + * + * @param currentGeneration state generation to keep. + * @param locations state paths. + */ + public void cleanupOldFiles(final long currentGeneration, Path[] locations) { + final String fileNameToKeep = prefix + currentGeneration + STATE_FILE_EXTENSION; for (Path location : locations) { logger.trace("cleanupOldFiles: cleaning up {}", location); Path stateLocation = location.resolve(STATE_DIR_NAME); try (Directory stateDir = newDirectory(stateLocation)) { for (String file : stateDir.listAll()) { - if (file.startsWith(prefix) && file.equals(currentStateFile) == false) { + if (file.startsWith(prefix) && file.equals(fileNameToKeep) == false) { deleteFileIgnoreExceptions(stateLocation, stateDir, file); } } @@ -346,31 +364,22 @@ private List findStateFilesByGeneration(final long generation, Path... loc } /** - * Tries to load the latest state from the given data-locations. It tries to load the latest state determined by - * the states version from one or more data directories and if none of the latest states can be loaded an exception - * is thrown to prevent accidentally loading a previous state and silently omitting the latest state. + * Tries to load the state of particular generation from the given data-locations. If any of data locations contain state files with + * given generation, state will be loaded from these state files. * - * @param logger a logger instance + * @param logger a logger instance. + * @param generation the generation to be loaded. * @param dataLocations the data-locations to try. - * @return the latest state or null if no state was found. + * @return the state of asked generation or null if no state was found. */ - public T loadLatestState(Logger logger, NamedXContentRegistry namedXContentRegistry, Path... dataLocations) throws IOException { - long maxStateId = findMaxStateId(prefix, dataLocations); - List stateFiles = findStateFilesByGeneration(maxStateId, dataLocations); - - if (maxStateId > -1 && stateFiles.isEmpty()) { - throw new IllegalStateException("unable to find state files with state id " + maxStateId + - " returned by findMaxStateId function, in data folders [" + - Arrays.stream(dataLocations).map(Path::toAbsolutePath). - map(Object::toString).collect(Collectors.joining(", ")) + - "], concurrent writes?"); - } + public T loadGeneration(Logger logger, NamedXContentRegistry namedXContentRegistry, long generation, Path... dataLocations) { + List stateFiles = findStateFilesByGeneration(generation, dataLocations); final List exceptions = new ArrayList<>(); for (Path stateFile : stateFiles) { try { T state = read(namedXContentRegistry, stateFile); - logger.trace("state id [{}] read from [{}]", maxStateId, stateFile.getFileName()); + logger.trace("state id [{}] read from [{}]", generation, stateFile.getFileName()); return state; } catch (Exception e) { exceptions.add(new IOException("failed to read " + stateFile.toAbsolutePath(), e)); @@ -388,6 +397,41 @@ public T loadLatestState(Logger logger, NamedXContentRegistry namedXContentRegis return null; } + + /** + * Tries to load the latest state from the given data-locations. + * + * @param logger a logger instance. + * @param dataLocations the data-locations to try. + * @return tuple of the latest state and generation. (-1, null) of no state is found. + */ + public Tuple loadLatestStateWithGeneration(Logger logger, NamedXContentRegistry namedXContentRegistry, Path... dataLocations) + throws IOException { + long generation = findMaxStateId(prefix, dataLocations); + T state = loadGeneration(logger, namedXContentRegistry, generation, dataLocations); + + if (generation > -1 && state == null) { + throw new IllegalStateException("unable to find state files with state id " + generation + + " returned by findMaxStateId function, in data folders [" + + Arrays.stream(dataLocations).map(Path::toAbsolutePath). + map(Object::toString).collect(Collectors.joining(", ")) + + "], concurrent writes?"); + } + return Tuple.tuple(state, generation); + } + + /** + * Tries to load the latest state from the given data-locations. + * + * @param logger a logger instance. + * @param dataLocations the data-locations to try. + * @return the latest state or null if no state was found. + */ + public T loadLatestState(Logger logger, NamedXContentRegistry namedXContentRegistry, Path... dataLocations) throws + IOException { + return loadLatestStateWithGeneration(logger, namedXContentRegistry, dataLocations).v1(); + } + /** * Deletes all meta state directories recursively for the given data locations * @param dataLocations the data location to delete diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java index 9377247488e7b..559416098a749 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java @@ -20,9 +20,12 @@ package org.elasticsearch.gateway; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.metadata.MetaState; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -31,14 +34,15 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.function.Predicate; /** - * Handles writing and loading both {@link MetaData} and {@link IndexMetaData} + * Handles writing and loading {@link MetaState}, {@link MetaData} and {@link IndexMetaData} */ -public class MetaStateService extends AbstractComponent { - +public class MetaStateService extends AbstractComponent implements IndexMetaDataWriter { private final NodeEnvironment nodeEnv; private final NamedXContentRegistry namedXContentRegistry; @@ -49,11 +53,60 @@ public MetaStateService(Settings settings, NodeEnvironment nodeEnv, NamedXConten } /** - * Loads the full state, which includes both the global state and all the indices - * meta state. + * Loads the full state, which includes both the global state and all the indices meta data.
+ * When loading, manifest file is consulted (represented by {@link MetaState} class), to load proper generations.
+ * If there is no manifest file on disk, this method fallbacks to BWC mode, where latest generation of global and indices + * metadata is loaded. Please note that currently where is no way to distinguish between manifest file being removed and manifest + * file was not yet created. It means that this method always fallbacks to BWC mode, if there is no manifest file. + * + * @return tuple of {@link MetaState} and {@link MetaData} with global metadata and indices metadata. If there is no state on disk, + * meta state with globalGeneration -1 and empty meta data is returned. + * @throws IOException if some IOException when loading files occurs or there is no metadata referenced by manifest file. */ - MetaData loadFullState() throws IOException { - MetaData globalMetaData = loadGlobalState(); + Tuple loadFullState() throws IOException { + final MetaState metaState = loadMetaState(); + if (metaState == null) { + MetaData globalMetaData = + MetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.nodeDataPaths()); + boolean isFreshStartup = globalMetaData == null; + if (isFreshStartup == false && Version.CURRENT.major >= 8) { + throw new IOException("failed to find manifest file, which is mandatory staring with ElasticSearch version 8.0"); + } + return loadFullStateBWC(); + } + final MetaData.Builder metaDataBuilder; + final MetaData globalMetaData = MetaData.FORMAT.loadGeneration(logger, namedXContentRegistry, metaState.getGlobalStateGeneration(), + nodeEnv.nodeDataPaths()); + if (globalMetaData != null) { + metaDataBuilder = MetaData.builder(globalMetaData); + } else { + throw new IOException("failed to find global metadata [generation: " + metaState.getGlobalStateGeneration() + "]"); + } + for (Map.Entry entry : metaState.getIndices().entrySet()) { + Index index = entry.getKey(); + long generation = entry.getValue(); + final String indexFolderName = index.getUUID(); + final IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadGeneration(logger, namedXContentRegistry, generation, + nodeEnv.resolveIndexFolder(indexFolderName)); + if (indexMetaData != null) { + metaDataBuilder.put(indexMetaData, false); + } else { + throw new IOException("failed to find metadata for existing index [location: " + indexFolderName + + ", generation: " + generation + "]"); + } + } + return new Tuple<>(metaState, metaDataBuilder.build()); + } + + /** + * Zen 1 BWC version of loading metadata from disk. See also {@link #loadFullState()} + */ + private Tuple loadFullStateBWC() throws IOException { + Map indices = new HashMap<>(); + Tuple metaDataAndGeneration = + MetaData.FORMAT.loadLatestStateWithGeneration(logger, namedXContentRegistry, nodeEnv.nodeDataPaths()); + MetaData globalMetaData = metaDataAndGeneration.v1(); + long globalStateGeneration = metaDataAndGeneration.v2(); MetaData.Builder metaDataBuilder; if (globalMetaData != null) { metaDataBuilder = MetaData.builder(globalMetaData); @@ -61,15 +114,20 @@ MetaData loadFullState() throws IOException { metaDataBuilder = MetaData.builder(); } for (String indexFolderName : nodeEnv.availableIndexFolders()) { - IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, - nodeEnv.resolveIndexFolder(indexFolderName)); + Tuple indexMetaDataAndGeneration = + IndexMetaData.FORMAT.loadLatestStateWithGeneration(logger, namedXContentRegistry, + nodeEnv.resolveIndexFolder(indexFolderName)); + IndexMetaData indexMetaData = indexMetaDataAndGeneration.v1(); + long generation = indexMetaDataAndGeneration.v2(); if (indexMetaData != null) { + indices.put(indexMetaData.getIndex(), generation); metaDataBuilder.put(indexMetaData, false); } else { logger.debug("[{}] failed to find metadata for existing index location", indexFolderName); } } - return metaDataBuilder.build(); + MetaState metaState = new MetaState(globalStateGeneration, indices); + return new Tuple<>(metaState, metaDataBuilder.build()); } /** @@ -80,6 +138,10 @@ public IndexMetaData loadIndexState(Index index) throws IOException { return IndexMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.indexPaths(index)); } + private MetaState loadMetaState() throws IOException { + return MetaState.FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.nodeDataPaths()); + } + /** * Loads all indices states available on disk */ @@ -87,15 +149,15 @@ List loadIndicesStates(Predicate excludeIndexPathIdsPredi List indexMetaDataList = new ArrayList<>(); for (String indexFolderName : nodeEnv.availableIndexFolders(excludeIndexPathIdsPredicate)) { assert excludeIndexPathIdsPredicate.test(indexFolderName) == false : - "unexpected folder " + indexFolderName + " which should have been excluded"; + "unexpected folder " + indexFolderName + " which should have been excluded"; IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, - nodeEnv.resolveIndexFolder(indexFolderName)); + nodeEnv.resolveIndexFolder(indexFolderName)); if (indexMetaData != null) { final String indexPathId = indexMetaData.getIndex().getUUID(); if (indexFolderName.equals(indexPathId)) { indexMetaDataList.add(indexMetaData); } else { - throw new IllegalStateException("[" + indexFolderName+ "] invalid index folder name, rename to [" + indexPathId + "]"); + throw new IllegalStateException("[" + indexFolderName + "] invalid index folder name, rename to [" + indexPathId + "]"); } } else { logger.debug("[{}] failed to find metadata for existing index location", indexFolderName); @@ -112,34 +174,115 @@ MetaData loadGlobalState() throws IOException { } /** - * Writes the index state. + * Writes manifest file (represented by {@link MetaState}) to disk. * + * @throws WriteStateException if exception when writing state occurs. See also {@link WriteStateException#isDirty()} + */ + public long writeMetaState(String reason, MetaState metaState) throws WriteStateException { + logger.trace("[_meta] writing state, reason [{}]", reason); + try { + long generation = MetaState.FORMAT.write(metaState, nodeEnv.nodeDataPaths()); + logger.trace("[_meta] state written (generation: {})", generation); + return generation; + } catch (WriteStateException ex) { + logger.warn("[_meta]: failed to write meta state", ex); + throw ex; + } + } + + /** + * Writes the index state. + *

* This method is public for testing purposes. + * + * @throws WriteStateException if exception when writing state occurs. {@link WriteStateException#isDirty()} will always return + * false, because new index state file is not yet referenced by manifest file. */ - public void writeIndex(String reason, IndexMetaData indexMetaData) throws IOException { + public long writeIndex(String reason, IndexMetaData indexMetaData) throws WriteStateException { final Index index = indexMetaData.getIndex(); logger.trace("[{}] writing state, reason [{}]", index, reason); try { - IndexMetaData.FORMAT.write(indexMetaData, - nodeEnv.indexPaths(indexMetaData.getIndex())); + long generation = IndexMetaData.FORMAT.write(indexMetaData, + nodeEnv.indexPaths(indexMetaData.getIndex())); logger.trace("[{}] state written", index); - } catch (Exception ex) { + return generation; + } catch (WriteStateException ex) { logger.warn(() -> new ParameterizedMessage("[{}]: failed to write index state", index), ex); - throw new IOException("failed to write state for [" + index + "]", ex); + ex.resetDirty(); + throw ex; } } /** * Writes the global state, *without* the indices states. + * + * @throws WriteStateException if exception when writing state occurs. {@link WriteStateException#isDirty()} will always return + * false, because new global state file is not yet referenced by manifest file. */ - void writeGlobalState(String reason, MetaData metaData) throws IOException { - logger.trace("[_global] writing state, reason [{}]", reason); + long writeGlobalState(String reason, MetaData metaData) throws WriteStateException { + logger.trace("[_global] writing state, reason [{}]", reason); try { - MetaData.FORMAT.write(metaData, nodeEnv.nodeDataPaths()); + long generation = MetaData.FORMAT.write(metaData, nodeEnv.nodeDataPaths()); logger.trace("[_global] state written"); - } catch (Exception ex) { + return generation; + } catch (WriteStateException ex) { logger.warn("[_global]: failed to write global state", ex); - throw new IOException("failed to write global state", ex); + ex.resetDirty(); + throw ex; } } -} + + /** + * Removes old state files in global state directory. + * + * @param currentGeneration current state generation to keep in the directory. + */ + void cleanupGlobalState(long currentGeneration) { + MetaData.FORMAT.cleanupOldFiles(currentGeneration, nodeEnv.nodeDataPaths()); + } + + /** + * Removes old state files in index directory. + * + * @param index index to perform clean up on. + * @param currentGeneration current state generation to keep in the index directory. + */ + public void cleanupIndex(Index index, long currentGeneration) { + IndexMetaData.FORMAT.cleanupOldFiles(currentGeneration, nodeEnv.indexPaths(index)); + } + + /** + * Removes old state files in meta state directory. + * + * @param currentGeneration current state generation to keep in the directory. + */ + public void cleanupMetaState(long currentGeneration) { + MetaState.FORMAT.cleanupOldFiles(currentGeneration, nodeEnv.nodeDataPaths()); + } + + /** + * Writes index metadata and updates manifest file accordingly. + */ + public void writeIndexAndUpdateMetaState(String reason, IndexMetaData metaData) throws IOException { + long generation = writeIndex(reason, metaData); + MetaState metaState = loadMetaState(); + Map indices = new HashMap<>(metaState.getIndices()); + indices.put(metaData.getIndex(), generation); + metaState = new MetaState(metaState.getGlobalStateGeneration(), indices); + long metaStateGeneration = writeMetaState(reason, metaState); + cleanupIndex(metaData.getIndex(), generation); + cleanupMetaState(metaStateGeneration); + } + + /** + * Writes global metadata and updates manifest file accordingly. + */ + public void writeGlobalStateAndUpdateMetaState(String reason, MetaData metaData) throws IOException { + long generation = writeGlobalState(reason, metaData); + MetaState metaState = loadMetaState(); + metaState = new MetaState(generation, metaState.getIndices()); + long metaStateGeneration = writeMetaState(reason, metaState); + cleanupGlobalState(generation); + cleanupMetaState(metaStateGeneration); + } +} \ No newline at end of file diff --git a/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java index 0a01c7cdabbe3..77a53f36f5f67 100644 --- a/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java +++ b/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java @@ -91,7 +91,7 @@ protected NodesGatewayMetaState newResponse(Request request, List "index.analysis.analyzer.test.tokenizer".equals(s) == false)).build(); - IndexMetaData.FORMAT.write(brokenMeta, services.indexPaths(brokenMeta.getIndex())); + metaStateService.writeIndexAndUpdateMetaState("broken metadata", brokenMeta); } internalCluster().fullRestart(); // ensureGreen(closedIndex) waits for the index to show up in the metadata @@ -495,11 +496,11 @@ public void testArchiveBrokenClusterSettings() throws Exception { } ClusterState state = client().admin().cluster().prepareState().get().getState(); MetaData metaData = state.getMetaData(); - for (NodeEnvironment nodeEnv : internalCluster().getInstances(NodeEnvironment.class)) { + for (MetaStateService metaStateService : internalCluster().getInstances(MetaStateService.class)) { MetaData brokenMeta = MetaData.builder(metaData).persistentSettings(Settings.builder() .put(metaData.persistentSettings()).put("this.is.unknown", true) .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), "broken").build()).build(); - MetaData.FORMAT.write(brokenMeta, nodeEnv.nodeDataPaths()); + metaStateService.writeGlobalStateAndUpdateMetaState("broken metadata", brokenMeta); } internalCluster().fullRestart(); ensureYellow("test"); // wait for state recovery diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java index 14f3c212c464c..13d084d990c1b 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java @@ -20,7 +20,6 @@ package org.elasticsearch.gateway; import org.elasticsearch.Version; -import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ESAllocationTestCase; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -37,138 +36,95 @@ import org.elasticsearch.plugins.MetaDataUpgrader; import org.elasticsearch.test.TestCustomMetaData; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; -import java.util.Iterator; +import java.util.HashSet; +import java.util.List; +import java.util.Map; import java.util.Set; -import static java.util.Collections.emptySet; -import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; -/** - * Test IndexMetaState for master and data only nodes return correct list of indices to write - * There are many parameters: - * - meta state is not in memory - * - meta state is in memory with old version/ new version - * - meta state is in memory with new version - * - version changed in cluster state event/ no change - * - node is data only node - * - node is master eligible - * for data only nodes: shard initializing on shard - */ public class GatewayMetaStateTests extends ESAllocationTestCase { - ClusterChangedEvent generateEvent(boolean initializing, boolean versionChanged, boolean masterEligible) { - //ridiculous settings to make sure we don't run into uninitialized because fo default - AllocationService strategy = createAllocationService(Settings.builder() - .put("cluster.routing.allocation.node_concurrent_recoveries", 100) - .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always") - .put("cluster.routing.allocation.cluster_concurrent_rebalance", 100) - .put("cluster.routing.allocation.node_initial_primaries_recoveries", 100) - .build()); - ClusterState newClusterState, previousClusterState; - MetaData metaDataOldClusterState = MetaData.builder() - .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(5).numberOfReplicas(2)) - .build(); - - RoutingTable routingTableOldClusterState = RoutingTable.builder() - .addAsNew(metaDataOldClusterState.index("test")) - .build(); + private ClusterState noIndexClusterState(boolean masterEligible) { + MetaData metaData = MetaData.builder().build(); + RoutingTable routingTable = RoutingTable.builder().build(); - // assign all shards - ClusterState init = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) - .metaData(metaDataOldClusterState) - .routingTable(routingTableOldClusterState) + return ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) + .metaData(metaData) + .routingTable(routingTable) .nodes(generateDiscoveryNodes(masterEligible)) .build(); - // new cluster state will have initializing shards on node 1 - RoutingTable routingTableNewClusterState = strategy.reroute(init, "reroute").routingTable(); - if (initializing == false) { - // pretend all initialized, nothing happened - ClusterState temp = ClusterState.builder(init).routingTable(routingTableNewClusterState) - .metaData(metaDataOldClusterState).build(); - routingTableNewClusterState = strategy.applyStartedShards(temp, temp.getRoutingNodes().shardsWithState(INITIALIZING)) - .routingTable(); - routingTableOldClusterState = routingTableNewClusterState; - - } else { - // nothing to do, we have one routing table with unassigned and one with initializing - } + } - // create new meta data either with version changed or not - MetaData metaDataNewClusterState = MetaData.builder() - .put(init.metaData().index("test"), versionChanged) + private ClusterState clusterStateWithUnassignedIndex(IndexMetaData indexMetaData, boolean masterEligible) { + MetaData metaData = MetaData.builder() + .put(indexMetaData, false) .build(); + RoutingTable routingTable = RoutingTable.builder() + .addAsNew(metaData.index("test")) + .build(); - // create the cluster states with meta data and routing tables as computed before - previousClusterState = ClusterState.builder(init) - .metaData(metaDataOldClusterState) - .routingTable(routingTableOldClusterState) + return ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) + .metaData(metaData) + .routingTable(routingTable) .nodes(generateDiscoveryNodes(masterEligible)) .build(); - newClusterState = ClusterState.builder(previousClusterState).routingTable(routingTableNewClusterState) - .metaData(metaDataNewClusterState).version(previousClusterState.getVersion() + 1).build(); - - ClusterChangedEvent event = new ClusterChangedEvent("test", newClusterState, previousClusterState); - assertThat(event.state().version(), equalTo(event.previousState().version() + 1)); - return event; } - ClusterChangedEvent generateCloseEvent(boolean masterEligible) { - //ridiculous settings to make sure we don't run into uninitialized because fo default + private ClusterState clusterStateWithAssignedIndex(IndexMetaData indexMetaData, boolean masterEligible) { AllocationService strategy = createAllocationService(Settings.builder() .put("cluster.routing.allocation.node_concurrent_recoveries", 100) .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always") .put("cluster.routing.allocation.cluster_concurrent_rebalance", 100) .put("cluster.routing.allocation.node_initial_primaries_recoveries", 100) .build()); - ClusterState newClusterState, previousClusterState; - MetaData metaDataIndexCreated = MetaData.builder() - .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(5).numberOfReplicas(2)) - .build(); - RoutingTable routingTableIndexCreated = RoutingTable.builder() - .addAsNew(metaDataIndexCreated.index("test")) - .build(); + ClusterState oldClusterState = clusterStateWithUnassignedIndex(indexMetaData, masterEligible); + RoutingTable routingTable = strategy.reroute(oldClusterState, "reroute").routingTable(); - // assign all shards - ClusterState init = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) - .metaData(metaDataIndexCreated) - .routingTable(routingTableIndexCreated) - .nodes(generateDiscoveryNodes(masterEligible)) - .build(); - RoutingTable routingTableInitializing = strategy.reroute(init, "reroute").routingTable(); - ClusterState temp = ClusterState.builder(init).routingTable(routingTableInitializing).build(); - RoutingTable routingTableStarted = strategy.applyStartedShards(temp, temp.getRoutingNodes().shardsWithState(INITIALIZING)) - .routingTable(); - - // create new meta data either with version changed or not - MetaData metaDataStarted = MetaData.builder() - .put(init.metaData().index("test"), true) + MetaData metaDataNewClusterState = MetaData.builder() + .put(oldClusterState.metaData().index("test"), false) .build(); - // create the cluster states with meta data and routing tables as computed before - MetaData metaDataClosed = MetaData.builder() + return ClusterState.builder(oldClusterState).routingTable(routingTable) + .metaData(metaDataNewClusterState).version(oldClusterState.getVersion() + 1).build(); + } + + private ClusterState clusterStateWithClosedIndex(IndexMetaData indexMetaData, boolean masterEligible) { + ClusterState oldClusterState = clusterStateWithAssignedIndex(indexMetaData, masterEligible); + + MetaData metaDataNewClusterState = MetaData.builder() .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).state(IndexMetaData.State.CLOSE) - .numberOfShards(5).numberOfReplicas(2)).version(metaDataStarted.version() + 1) + .numberOfShards(5).numberOfReplicas(2)) + .version(oldClusterState.metaData().version() + 1) .build(); - previousClusterState = ClusterState.builder(init) - .metaData(metaDataStarted) - .routingTable(routingTableStarted) - .nodes(generateDiscoveryNodes(masterEligible)) + RoutingTable routingTable = RoutingTable.builder() + .addAsNew(metaDataNewClusterState.index("test")) .build(); - newClusterState = ClusterState.builder(previousClusterState) - .routingTable(routingTableIndexCreated) - .metaData(metaDataClosed) - .version(previousClusterState.getVersion() + 1).build(); - - ClusterChangedEvent event = new ClusterChangedEvent("test", newClusterState, previousClusterState); - assertThat(event.state().version(), equalTo(event.previousState().version() + 1)); - return event; + + return ClusterState.builder(oldClusterState).routingTable(routingTable) + .metaData(metaDataNewClusterState).version(oldClusterState.getVersion() + 1).build(); + } + + private ClusterState clusterStateWithJustOpenedIndex(IndexMetaData indexMetaData, boolean masterEligible) { + ClusterState oldClusterState = clusterStateWithClosedIndex(indexMetaData, masterEligible); + + MetaData metaDataNewClusterState = MetaData.builder() + .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).state(IndexMetaData.State.OPEN) + .numberOfShards(5).numberOfReplicas(2)) + .version(oldClusterState.metaData().version() + 1) + .build(); + + return ClusterState.builder(oldClusterState) + .metaData(metaDataNewClusterState).version(oldClusterState.getVersion() + 1).build(); } private DiscoveryNodes.Builder generateDiscoveryNodes(boolean masterEligible) { @@ -177,80 +133,162 @@ private DiscoveryNodes.Builder generateDiscoveryNodes(boolean masterEligible) { .add(newNode("master_node", MASTER_DATA_ROLES)).localNodeId("node1").masterNodeId(masterEligible ? "node1" : "master_node"); } - public void assertState(ClusterChangedEvent event, - boolean stateInMemory, - boolean expectMetaData) throws Exception { - MetaData inMemoryMetaData = null; - Set oldIndicesList = emptySet(); - if (stateInMemory) { - inMemoryMetaData = event.previousState().metaData(); - oldIndicesList = GatewayMetaState.getRelevantIndices(event.previousState(), event.previousState(), oldIndicesList); - } - Set newIndicesList = GatewayMetaState.getRelevantIndices(event.state(),event.previousState(), oldIndicesList); - // third, get the actual write info - Iterator indices = GatewayMetaState.resolveStatesToBeWritten(oldIndicesList, newIndicesList, - inMemoryMetaData, event.state().metaData()).iterator(); - - if (expectMetaData) { - assertThat(indices.hasNext(), equalTo(true)); - assertThat(indices.next().getNewMetaData().getIndex().getName(), equalTo("test")); - assertThat(indices.hasNext(), equalTo(false)); + private Set randomPrevWrittenIndices(IndexMetaData indexMetaData) { + if (randomBoolean()) { + return Collections.singleton(indexMetaData.getIndex()); } else { - assertThat(indices.hasNext(), equalTo(false)); + return Collections.emptySet(); } } - public void testVersionChangeIsAlwaysWritten() throws Exception { - // test that version changes are always written - boolean initializing = randomBoolean(); - boolean versionChanged = true; - boolean stateInMemory = randomBoolean(); - boolean masterEligible = randomBoolean(); - boolean expectMetaData = true; - ClusterChangedEvent event = generateEvent(initializing, versionChanged, masterEligible); - assertState(event, stateInMemory, expectMetaData); + private IndexMetaData createIndexMetaData(String name) { + return IndexMetaData.builder(name). + settings(settings(Version.CURRENT)). + numberOfShards(5). + numberOfReplicas(2). + build(); } - public void testNewShardsAlwaysWritten() throws Exception { - // make sure new shards on data only node always written - boolean initializing = true; - boolean versionChanged = randomBoolean(); - boolean stateInMemory = randomBoolean(); - boolean masterEligible = false; - boolean expectMetaData = true; - ClusterChangedEvent event = generateEvent(initializing, versionChanged, masterEligible); - assertState(event, stateInMemory, expectMetaData); + public void testGetRelevantIndices_master_unassignedShards() { + IndexMetaData indexMetaData = createIndexMetaData("test"); + Set indices = GatewayMetaState.getRelevantIndices( + clusterStateWithUnassignedIndex(indexMetaData, true), + noIndexClusterState(true), + randomPrevWrittenIndices(indexMetaData)); + assertThat(indices.size(), equalTo(1)); } - public void testAllUpToDateNothingWritten() throws Exception { - // make sure state is not written again if we wrote already - boolean initializing = false; - boolean versionChanged = false; - boolean stateInMemory = true; - boolean masterEligible = randomBoolean(); - boolean expectMetaData = false; - ClusterChangedEvent event = generateEvent(initializing, versionChanged, masterEligible); - assertState(event, stateInMemory, expectMetaData); + public void testGetRelevantIndices_data_unassignedShards() { + IndexMetaData indexMetaData = createIndexMetaData("test"); + Set indices = GatewayMetaState.getRelevantIndices( + clusterStateWithUnassignedIndex(indexMetaData, false), + noIndexClusterState(false), + randomPrevWrittenIndices(indexMetaData)); + assertThat(indices.size(), equalTo(0)); } - public void testNoWriteIfNothingChanged() throws Exception { - boolean initializing = false; - boolean versionChanged = false; - boolean stateInMemory = true; + public void testGetRelevantIndices_assignedShards() { + IndexMetaData indexMetaData = createIndexMetaData("test"); boolean masterEligible = randomBoolean(); - boolean expectMetaData = false; - ClusterChangedEvent event = generateEvent(initializing, versionChanged, masterEligible); - ClusterChangedEvent newEventWithNothingChanged = new ClusterChangedEvent("test cluster state", event.state(), event.state()); - assertState(newEventWithNothingChanged, stateInMemory, expectMetaData); + Set indices = GatewayMetaState.getRelevantIndices( + clusterStateWithAssignedIndex(indexMetaData, masterEligible), + clusterStateWithUnassignedIndex(indexMetaData, masterEligible), + randomPrevWrittenIndices(indexMetaData)); + assertThat(indices.size(), equalTo(1)); } - public void testWriteClosedIndex() throws Exception { - // test that the closing of an index is written also on data only node - boolean masterEligible = randomBoolean(); - boolean expectMetaData = true; - boolean stateInMemory = true; - ClusterChangedEvent event = generateCloseEvent(masterEligible); - assertState(event, stateInMemory, expectMetaData); + public void testGetRelevantIndices_data_isClosedPrevWritten() { + IndexMetaData indexMetaData = createIndexMetaData("test"); + Set indices = GatewayMetaState.getRelevantIndices( + clusterStateWithClosedIndex(indexMetaData, false), + clusterStateWithAssignedIndex(indexMetaData, false), + Collections.singleton(indexMetaData.getIndex())); + assertThat(indices.size(), equalTo(1)); + } + + public void testGetRelevantIndices_data_isClosedPrevNotWritten() { + IndexMetaData indexMetaData = createIndexMetaData("test"); + Set indices = GatewayMetaState.getRelevantIndices( + clusterStateWithJustOpenedIndex(indexMetaData, false), + clusterStateWithClosedIndex(indexMetaData, false), + Collections.emptySet()); + assertThat(indices.size(), equalTo(0)); + } + + public void testGetRelevantIndices_data_wasClosedPrevWritten() { + IndexMetaData indexMetaData = createIndexMetaData("test"); + Set indices = GatewayMetaState.getRelevantIndices( + clusterStateWithJustOpenedIndex(indexMetaData, false), + clusterStateWithClosedIndex(indexMetaData, false), + Collections.singleton(indexMetaData.getIndex())); + assertThat(indices.size(), equalTo(1)); + } + + public void testResolveStatesToBeWritten() throws WriteStateException { + Map indices = new HashMap<>(); + Set relevantIndices = new HashSet<>(); + + IndexMetaData removedIndex = createIndexMetaData("removed_index"); + indices.put(removedIndex.getIndex(), 1L); + + IndexMetaData versionChangedIndex = createIndexMetaData("version_changed_index"); + indices.put(versionChangedIndex.getIndex(), 2L); + relevantIndices.add(versionChangedIndex.getIndex()); + + IndexMetaData notChangedIndex = createIndexMetaData("not_changed_index"); + indices.put(notChangedIndex.getIndex(), 3L); + relevantIndices.add(notChangedIndex.getIndex()); + + IndexMetaData newIndex = createIndexMetaData("new_index"); + relevantIndices.add(newIndex.getIndex()); + + MetaData oldMetaData = MetaData.builder() + .put(removedIndex, false) + .put(versionChangedIndex, false) + .put(notChangedIndex, false) + .build(); + + MetaData newMetaData = MetaData.builder() + .put(versionChangedIndex, true) + .put(notChangedIndex, false) + .put(newIndex, false) + .build(); + + IndexMetaData newVersionChangedIndex = newMetaData.index(versionChangedIndex.getIndex()); + + List actions = + GatewayMetaState.resolveIndexMetaDataActions(indices, relevantIndices, oldMetaData, newMetaData); + + List cleanupActions = new ArrayList<>(); + + assertThat(actions, hasSize(3)); + + for (GatewayMetaState.IndexMetaDataAction action : actions) { + if (action instanceof GatewayMetaState.KeepPreviousGeneration) { + assertThat(action.getIndex(), equalTo(notChangedIndex.getIndex())); + assertThat(action.execute(null, cleanupActions), equalTo(3L)); + } + if (action instanceof GatewayMetaState.WriteNewIndexMetaData) { + assertThat(action.getIndex(), equalTo(newIndex.getIndex())); + action.execute(new IndexMetaDataWriter() { + @Override + public long writeIndex(String reason, IndexMetaData indexMetaData) { + assertThat(reason, equalTo("freshly created")); + assertThat(indexMetaData, equalTo(newIndex)); + return 0L; + } + + @Override + public void cleanupIndex(Index index, long currentGeneration) { + assertThat(index, equalTo(newIndex.getIndex())); + assertThat(currentGeneration, equalTo(0L)); + } + }, cleanupActions); + } + if (action instanceof GatewayMetaState.WriteChangedIndexMetaData) { + assertThat(action.getIndex(), equalTo(newVersionChangedIndex.getIndex())); + action.execute(new IndexMetaDataWriter() { + @Override + public long writeIndex(String reason, IndexMetaData indexMetaData) { + assertThat(reason, containsString(Long.toString(versionChangedIndex.getVersion()))); + assertThat(reason, containsString(Long.toString(newVersionChangedIndex.getVersion()))); + assertThat(indexMetaData, equalTo(newVersionChangedIndex)); + return 3L; + } + + @Override + public void cleanupIndex(Index index, long currentGeneration) { + assertThat(index, equalTo(newVersionChangedIndex.getIndex())); + assertThat(currentGeneration, equalTo(3L)); + } + }, cleanupActions); + } + } + + assertThat(cleanupActions, hasSize(2)); + for (Runnable cleanupAction : cleanupActions) { + cleanupAction.run(); + } } public void testAddCustomMetaDataOnUpgrade() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java b/server/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java index d8433963e5226..e89f204917c68 100644 --- a/server/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java @@ -46,7 +46,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.file.DirectoryStream; @@ -71,11 +70,11 @@ public class MetaDataStateFormatTests extends ESTestCase { /** * Ensure we can read a pre-generated cluster state. */ - public void testReadClusterState() throws URISyntaxException, IOException { + public void testReadClusterState() throws IOException { final MetaDataStateFormat format = new MetaDataStateFormat("global-") { @Override - public void toXContent(XContentBuilder builder, MetaData state) throws IOException { + public void toXContent(XContentBuilder builder, MetaData state) { fail("this test doesn't write"); } @@ -473,7 +472,6 @@ private enum FailureMode { this.failureMode = FailureMode.NO_FAILURES; } - @Override public void toXContent(XContentBuilder builder, DummyState state) throws IOException { state.toXContent(builder, null); @@ -484,7 +482,6 @@ public DummyState fromXContent(XContentParser parser) throws IOException { return new DummyState().parse(parser); } - public void noFailures() { this.failureMode = FailureMode.NO_FAILURES; } diff --git a/server/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java b/server/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java index f2bacc154bf46..909860a54c7f2 100644 --- a/server/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java @@ -183,7 +183,7 @@ private boolean indexDirectoryExists(String nodeName, Index index) { private ImmutableOpenMap getIndicesMetaDataOnNode(String nodeName) throws Exception { GatewayMetaState nodeMetaState = ((InternalTestCluster) cluster()).getInstance(GatewayMetaState.class, nodeName); - MetaData nodeMetaData = nodeMetaState.loadMetaState(); + MetaData nodeMetaData = nodeMetaState.loadMetaData(); return nodeMetaData.getIndices(); } } diff --git a/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java b/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java index d0bf02e3c4e1f..d2d49ce37cac8 100644 --- a/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java @@ -21,12 +21,19 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.metadata.MetaState; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.Index; import org.elasticsearch.test.ESTestCase; +import java.io.IOException; +import java.util.HashMap; + +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.nullValue; public class MetaStateServiceTests extends ESTestCase { @@ -81,7 +88,48 @@ public void testWriteGlobalStateWithIndexAndNoIndexIsLoaded() throws Exception { } } - public void testLoadGlobal() throws Exception { + public void testLoadFullStateBWC() throws Exception { + try (NodeEnvironment env = newNodeEnvironment()) { + MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry()); + + IndexMetaData indexMetaData = IndexMetaData.builder("test1").settings(indexSettings).build(); + MetaData metaData = MetaData.builder() + .persistentSettings(Settings.builder().put("test1", "value1").build()) + .put(indexMetaData, true) + .build(); + + long globalGeneration = metaStateService.writeGlobalState("test_write", metaData); + long indexGeneration = metaStateService.writeIndex("test_write", indexMetaData); + + Tuple stateAndData = metaStateService.loadFullState(); + MetaState metaState = stateAndData.v1(); + assertThat(metaState.getGlobalStateGeneration(), equalTo(globalGeneration)); + assertThat(metaState.getIndices(), hasKey(indexMetaData.getIndex())); + assertThat(metaState.getIndices().get(indexMetaData.getIndex()), equalTo(indexGeneration)); + + MetaData loadedMetaData = stateAndData.v2(); + assertThat(loadedMetaData.persistentSettings(), equalTo(metaData.persistentSettings())); + assertThat(loadedMetaData.hasIndex("test1"), equalTo(true)); + assertThat(loadedMetaData.index("test1"), equalTo(indexMetaData)); + } + } + + public void testLoadEmptyState() throws IOException { + try (NodeEnvironment env = newNodeEnvironment()) { + MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry()); + Tuple stateAndData = metaStateService.loadFullState(); + + MetaState metaState = stateAndData.v1(); + assertThat(metaState.getGlobalStateGeneration(), equalTo(-1L)); + assertThat(metaState.getIndices().entrySet(), empty()); + + MetaData metaData = stateAndData.v2(); + MetaData emptyMetaData = MetaData.builder().build(); + assertTrue(MetaData.isGlobalStateEquals(metaData, emptyMetaData)); + } + } + + public void testLoadFullStateAndUpdate() throws IOException { try (NodeEnvironment env = newNodeEnvironment()) { MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry()); @@ -91,13 +139,45 @@ public void testLoadGlobal() throws Exception { .put(index, true) .build(); - metaStateService.writeGlobalState("test_write", metaData); - metaStateService.writeIndex("test_write", index); + long globalGeneration = metaStateService.writeGlobalState("first global state write", metaData); + long indexGeneration = metaStateService.writeIndex("first index state write", index); + + MetaState metaState = new MetaState(globalGeneration, new HashMap() {{ + put(index.getIndex(), indexGeneration); + }}); + + metaStateService.writeMetaState("first meta state write", metaState); + + MetaData newMetaData = MetaData.builder() + .persistentSettings(Settings.builder().put("test1", "value2").build()) + .put(index, true) + .build(); + globalGeneration = metaStateService.writeGlobalState("second global state write", newMetaData); + + Tuple stateAndData = metaStateService.loadFullState(); + assertThat(stateAndData.v1(), equalTo(metaState)); + + MetaData loadedMetaData = stateAndData.v2(); + assertThat(loadedMetaData.persistentSettings(), equalTo(metaData.persistentSettings())); + assertThat(loadedMetaData.hasIndex("test1"), equalTo(true)); + assertThat(loadedMetaData.index("test1"), equalTo(index)); + + metaState = new MetaState(globalGeneration, new HashMap() {{ + put(index.getIndex(), indexGeneration); + }}); + + long metaStateGeneration = metaStateService.writeMetaState("test", metaState); + metaStateService.cleanupGlobalState(globalGeneration); + metaStateService.cleanupIndex(index.getIndex(), indexGeneration); + metaStateService.cleanupMetaState(metaStateGeneration); + + stateAndData = metaStateService.loadFullState(); + assertThat(stateAndData.v1(), equalTo(metaState)); - MetaData loadedState = metaStateService.loadFullState(); - assertThat(loadedState.persistentSettings(), equalTo(metaData.persistentSettings())); - assertThat(loadedState.hasIndex("test1"), equalTo(true)); - assertThat(loadedState.index("test1"), equalTo(index)); + loadedMetaData = stateAndData.v2(); + assertThat(loadedMetaData.persistentSettings(), equalTo(newMetaData.persistentSettings())); + assertThat(loadedMetaData.hasIndex("test1"), equalTo(true)); + assertThat(loadedMetaData.index("test1"), equalTo(index)); } } } diff --git a/server/src/test/java/org/elasticsearch/gateway/MetaStateTests.java b/server/src/test/java/org/elasticsearch/gateway/MetaStateTests.java new file mode 100644 index 0000000000000..dbbfe08f3d7da --- /dev/null +++ b/server/src/test/java/org/elasticsearch/gateway/MetaStateTests.java @@ -0,0 +1,89 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gateway; + +import org.elasticsearch.cluster.metadata.MetaState; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.index.Index; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; + +public class MetaStateTests extends ESTestCase { + + private MetaState copyState(MetaState state, boolean introduceErrors){ + long generation = state.getGlobalStateGeneration(); + Map indices = new HashMap<>(state.getIndices()); + if (introduceErrors) { + if (randomBoolean()){ + generation = generation + 1; + } else { + indices.remove(randomFrom(indices.keySet())); + } + } + return new MetaState(generation, indices); + } + + private MetaState createRandomState() { + long generation = randomNonNegativeLong(); + Map indices = new HashMap<>(); + for (int i=0; i Date: Wed, 31 Oct 2018 13:25:16 +0300 Subject: [PATCH 02/57] Fix code review issues --- .../gateway/GatewayMetaState.java | 4 +-- .../gateway/MetaDataStateFormat.java | 13 +++++++--- .../gateway/MetaStateService.java | 26 +++++++------------ .../gateway/WriteStateException.java | 6 +---- 4 files changed, 23 insertions(+), 26 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index 250261be6158f..d5467bb20d0cd 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -114,7 +114,6 @@ private void maybeUpgradeMetaData(MetaDataIndexUpgradeService metaDataIndexUpgra List cleanupActions = new ArrayList<>(); // If globalStateGeneration is non-negative, it means we should have some metadata on disk // Always re-write it even if upgrade plugins do not upgrade it, to be sure it's properly persisted on all data path - Map indices = new HashMap<>(metaState.getIndices()); final MetaData upgradedMetaData = upgradeMetaData(metaData, metaDataIndexUpgradeService, metaDataUpgrader); if (MetaData.isGlobalStateEquals(metaData, upgradedMetaData) == false) { @@ -125,6 +124,7 @@ private void maybeUpgradeMetaData(MetaDataIndexUpgradeService metaDataIndexUpgra final long currentGlobalStateGeneration = globalStateGeneration; cleanupActions.add(() -> metaStateService.cleanupGlobalState(currentGlobalStateGeneration)); + Map indices = new HashMap<>(); for (IndexMetaData indexMetaData : upgradedMetaData) { long generation; if (metaData.hasIndexMetaData(indexMetaData) == false) { @@ -143,7 +143,7 @@ private void maybeUpgradeMetaData(MetaDataIndexUpgradeService metaDataIndexUpgra performCleanup(cleanupActions); } } catch (Exception e) { - logger.error("failed to read local state, exiting...", e); + logger.error("failed to read or re-write local state, exiting...", e); throw e; } } diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java b/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java index 41a42ec4662b1..2474e570bb60a 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java @@ -209,7 +209,7 @@ public final long write(final T state, final Path... locations) throws WriteStat } assert maxStateId >= 0 : "maxStateId must be positive but was: [" + maxStateId + "]"; - final String fileName = prefix + maxStateId + STATE_FILE_EXTENSION; + final String fileName = getStateFileName(maxStateId); final String tmpFileName = fileName + ".tmp"; List> directories = new ArrayList<>(); @@ -291,6 +291,9 @@ protected Directory newDirectory(Path dir) throws IOException { return new SimpleFSDirectory(dir); } + /** + * Whether to perform autoCleanup of old state files after successful {@link #write(Object, Path...)}. + */ protected boolean autoCleanup() { return true; } @@ -302,7 +305,7 @@ protected boolean autoCleanup() { * @param locations state paths. */ public void cleanupOldFiles(final long currentGeneration, Path[] locations) { - final String fileNameToKeep = prefix + currentGeneration + STATE_FILE_EXTENSION; + final String fileNameToKeep = getStateFileName(currentGeneration); for (Path location : locations) { logger.trace("cleanupOldFiles: cleaning up {}", location); Path stateLocation = location.resolve(STATE_DIR_NAME); @@ -351,7 +354,7 @@ private List findStateFilesByGeneration(final long generation, Path... loc return files; } - final String fileName = prefix + generation + STATE_FILE_EXTENSION; + final String fileName = getStateFileName(generation); for (Path dataLocation : locations) { final Path stateFilePath = dataLocation.resolve(STATE_DIR_NAME).resolve(fileName); if (Files.exists(stateFilePath)) { @@ -363,6 +366,10 @@ private List findStateFilesByGeneration(final long generation, Path... loc return files; } + private String getStateFileName(long generation) { + return prefix + generation + STATE_FILE_EXTENSION; + } + /** * Tries to load the state of particular generation from the given data-locations. If any of data locations contain state files with * given generation, state will be loaded from these state files. diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java index 559416098a749..122218b965da9 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java @@ -19,7 +19,6 @@ package org.elasticsearch.gateway; -import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; @@ -56,7 +55,7 @@ public MetaStateService(Settings settings, NodeEnvironment nodeEnv, NamedXConten * Loads the full state, which includes both the global state and all the indices meta data.
* When loading, manifest file is consulted (represented by {@link MetaState} class), to load proper generations.
* If there is no manifest file on disk, this method fallbacks to BWC mode, where latest generation of global and indices - * metadata is loaded. Please note that currently where is no way to distinguish between manifest file being removed and manifest + * metadata is loaded. Please note that currently there is no way to distinguish between manifest file being removed and manifest * file was not yet created. It means that this method always fallbacks to BWC mode, if there is no manifest file. * * @return tuple of {@link MetaState} and {@link MetaData} with global metadata and indices metadata. If there is no state on disk, @@ -66,12 +65,6 @@ public MetaStateService(Settings settings, NodeEnvironment nodeEnv, NamedXConten Tuple loadFullState() throws IOException { final MetaState metaState = loadMetaState(); if (metaState == null) { - MetaData globalMetaData = - MetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.nodeDataPaths()); - boolean isFreshStartup = globalMetaData == null; - if (isFreshStartup == false && Version.CURRENT.major >= 8) { - throw new IOException("failed to find manifest file, which is mandatory staring with ElasticSearch version 8.0"); - } return loadFullStateBWC(); } final MetaData.Builder metaDataBuilder; @@ -107,6 +100,12 @@ private Tuple loadFullStateBWC() throws IOException { MetaData.FORMAT.loadLatestStateWithGeneration(logger, namedXContentRegistry, nodeEnv.nodeDataPaths()); MetaData globalMetaData = metaDataAndGeneration.v1(); long globalStateGeneration = metaDataAndGeneration.v2(); + boolean isFreshStartup = globalMetaData == null; + + if (isFreshStartup) { + assert Version.CURRENT.major < 8 : "failed to find manifest file, which is mandatory staring with Elasticsearch version 8.0"; + } + MetaData.Builder metaDataBuilder; if (globalMetaData != null) { metaDataBuilder = MetaData.builder(globalMetaData); @@ -185,8 +184,7 @@ public long writeMetaState(String reason, MetaState metaState) throws WriteState logger.trace("[_meta] state written (generation: {})", generation); return generation; } catch (WriteStateException ex) { - logger.warn("[_meta]: failed to write meta state", ex); - throw ex; + throw new WriteStateException(ex.isDirty(), "[_meta]: failed to write meta state", ex); } } @@ -207,9 +205,7 @@ public long writeIndex(String reason, IndexMetaData indexMetaData) throws WriteS logger.trace("[{}] state written", index); return generation; } catch (WriteStateException ex) { - logger.warn(() -> new ParameterizedMessage("[{}]: failed to write index state", index), ex); - ex.resetDirty(); - throw ex; + throw new WriteStateException(false, "[" + index + "]: failed to write index state", ex); } } @@ -226,9 +222,7 @@ long writeGlobalState(String reason, MetaData metaData) throws WriteStateExcepti logger.trace("[_global] state written"); return generation; } catch (WriteStateException ex) { - logger.warn("[_global]: failed to write global state", ex); - ex.resetDirty(); - throw ex; + throw new WriteStateException(false, "[_global]: failed to write global state", ex); } } diff --git a/server/src/main/java/org/elasticsearch/gateway/WriteStateException.java b/server/src/main/java/org/elasticsearch/gateway/WriteStateException.java index 869ec87742654..aeb9bffc6711d 100644 --- a/server/src/main/java/org/elasticsearch/gateway/WriteStateException.java +++ b/server/src/main/java/org/elasticsearch/gateway/WriteStateException.java @@ -24,7 +24,7 @@ * This exception is thrown when there is a problem of writing state to disk. */ public class WriteStateException extends IOException { - private boolean dirty; + private final boolean dirty; WriteStateException(boolean dirty, String message, Exception cause) { super(message, cause); @@ -38,8 +38,4 @@ public class WriteStateException extends IOException { public boolean isDirty() { return dirty; } - - public void resetDirty() { - this.dirty = false; - } } From 5dcb6f52a10192fcda267568bb3452c2b46ae6bb Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Wed, 7 Nov 2018 21:14:34 +0300 Subject: [PATCH 03/57] MetaState -> Manifest --- .../{MetaState.java => Manifest.java} | 26 ++++----- .../gateway/GatewayMetaState.java | 28 ++++----- .../gateway/MetaStateService.java | 58 +++++++++---------- .../admin/indices/create/CreateIndexIT.java | 2 +- .../gateway/GatewayIndexStateIT.java | 6 +- ...MetaStateTests.java => ManifestTests.java} | 26 ++++----- .../gateway/MetaStateServiceTests.java | 34 +++++------ 7 files changed, 90 insertions(+), 90 deletions(-) rename server/src/main/java/org/elasticsearch/cluster/metadata/{MetaState.java => Manifest.java} (86%) rename server/src/test/java/org/elasticsearch/gateway/{MetaStateTests.java => ManifestTests.java} (79%) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaState.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java similarity index 86% rename from server/src/main/java/org/elasticsearch/cluster/metadata/MetaState.java rename to server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java index 98d2a2dfa7116..fbf2ae2049734 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaState.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java @@ -43,11 +43,11 @@ * Global metadata generation could be obtained by calling {@link #getGlobalStateGeneration()}. * Index metadata generation could be obtained by calling {@link #getIndices()}. */ -public class MetaState implements ToXContentFragment { +public class Manifest implements ToXContentFragment { private final long globalStateGeneration; private final Map indices; - public MetaState(long globalStateGeneration, Map indices) { + public Manifest(long globalStateGeneration, Map indices) { this.globalStateGeneration = globalStateGeneration; this.indices = indices; } @@ -70,9 +70,9 @@ public Map getIndices() { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - MetaState metaState = (MetaState) o; - return globalStateGeneration == metaState.globalStateGeneration && - Objects.equals(indices, metaState.indices); + Manifest manifest = (Manifest) o; + return globalStateGeneration == manifest.globalStateGeneration && + Objects.equals(indices, manifest.indices); } @Override @@ -83,7 +83,7 @@ public int hashCode() { private static final String META_STATE_FILE_PREFIX = "meta-"; private static final ToXContent.Params METASTATE_FORMAT_PARAMS = new ToXContent.MapParams(Collections.singletonMap("binary", "true")); - public static final MetaDataStateFormat FORMAT = new MetaDataStateFormat(META_STATE_FILE_PREFIX) { + public static final MetaDataStateFormat FORMAT = new MetaDataStateFormat(META_STATE_FILE_PREFIX) { @Override protected boolean autoCleanup() { @@ -91,13 +91,13 @@ protected boolean autoCleanup() { } @Override - public void toXContent(XContentBuilder builder, MetaState state) throws IOException { + public void toXContent(XContentBuilder builder, Manifest state) throws IOException { state.toXContent(builder, METASTATE_FORMAT_PARAMS); } @Override - public MetaState fromXContent(XContentParser parser) throws IOException { - return MetaState.fromXContent(parser); + public Manifest fromXContent(XContentParser parser) throws IOException { + return Manifest.fromXContent(parser); } }; @@ -131,17 +131,17 @@ private static Map indicies(Object[] generationAndListOfIndexEntrie return listOfIndices.stream().collect(Collectors.toMap(IndexEntry::getIndex, IndexEntry::getGeneration)); } - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - "state", + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "manifest", generationAndListOfIndexEntries -> - new MetaState(generation(generationAndListOfIndexEntries), indicies(generationAndListOfIndexEntries))); + new Manifest(generation(generationAndListOfIndexEntries), indicies(generationAndListOfIndexEntries))); static { PARSER.declareLong(ConstructingObjectParser.constructorArg(), GENERATION_PARSE_FIELD); PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), IndexEntry.INDEX_ENTRY_PARSER, INDICES_PARSE_FIELD); } - public static MetaState fromXContent(XContentParser parser) throws IOException { + public static Manifest fromXContent(XContentParser parser) throws IOException { return PARSER.parse(parser, null); } diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index d5467bb20d0cd..688a47658b989 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -25,9 +25,9 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateApplier; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.Manifest; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService; -import org.elasticsearch.cluster.metadata.MetaState; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.ShardRouting; @@ -75,7 +75,7 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateA @Nullable //there is happens-before order between subsequent applyClusterState calls, hence no volatile modifier - private Tuple ourState; + private Tuple ourState; public GatewayMetaState(Settings settings, NodeEnvironment nodeEnv, MetaStateService metaStateService, MetaDataIndexUpgradeService metaDataIndexUpgradeService, MetaDataUpgrader metaDataUpgrader) throws IOException { @@ -101,14 +101,14 @@ private void maybeUpgradeMetaData(MetaDataIndexUpgradeService metaDataIndexUpgra throws IOException { if (DiscoveryNode.isMasterNode(settings) || DiscoveryNode.isDataNode(settings)) { try { - final Tuple metaStateAndData = metaStateService.loadFullState(); - final MetaState metaState = metaStateAndData.v1(); + final Tuple metaStateAndData = metaStateService.loadFullState(); + final Manifest manifest = metaStateAndData.v1(); final MetaData metaData = metaStateAndData.v2(); // We finished global state validation and successfully checked all indices for backward compatibility // and found no non-upgradable indices, which means the upgrade can continue. // Now it's safe to overwrite global and index metadata. - long globalStateGeneration = metaState.getGlobalStateGeneration(); + long globalStateGeneration = manifest.getGlobalStateGeneration(); if (globalStateGeneration != -1) { List cleanupActions = new ArrayList<>(); @@ -138,7 +138,7 @@ private void maybeUpgradeMetaData(MetaDataIndexUpgradeService metaDataIndexUpgra } final long metaStateGeneration = - metaStateService.writeMetaState("startup", new MetaState(globalStateGeneration, indices)); + metaStateService.writeManifest("startup", new Manifest(globalStateGeneration, indices)); cleanupActions.add(() -> metaStateService.cleanupMetaState(metaStateGeneration)); performCleanup(cleanupActions); } @@ -194,7 +194,7 @@ public void applyClusterState(ClusterChangedEvent event) { * @throws IOException if IOException occurs. It's recommended for the callers of this method to handle {@link WriteStateException}, * which is subclass of {@link IOException} explicitly. See also {@link WriteStateException#isDirty()}. */ - private Tuple updateMetaData(ClusterChangedEvent event) throws IOException { + private Tuple updateMetaData(ClusterChangedEvent event) throws IOException { ClusterState newState = event.state(); ClusterState previousState = event.previousState(); MetaData newMetaData = newState.metaData(); @@ -202,10 +202,10 @@ private Tuple updateMetaData(ClusterChangedEvent event) thr List cleanupActions = new ArrayList<>(); long globalStateGeneration = writeGlobalState(newMetaData, cleanupActions); Map newIndices = writeIndicesMetadata(newState, previousState, cleanupActions); - MetaState metaState = new MetaState(globalStateGeneration, newIndices); - writeMetaState(metaState, cleanupActions); + Manifest manifest = new Manifest(globalStateGeneration, newIndices); + writeManifest(manifest, cleanupActions); performCleanup(cleanupActions); - return new Tuple<>(metaState, newMetaData); + return new Tuple<>(manifest, newMetaData); } private void performCleanup(List cleanupActions) { @@ -214,9 +214,9 @@ private void performCleanup(List cleanupActions) { } } - private void writeMetaState(MetaState metaState, List cleanupActions) throws IOException { - if (metaState.equals(ourState.v1()) == false) { - long generation = metaStateService.writeMetaState("changed", metaState); + private void writeManifest(Manifest manifest, List cleanupActions) throws IOException { + if (manifest.equals(ourState.v1()) == false) { + long generation = metaStateService.writeManifest("changed", manifest); cleanupActions.add(() -> metaStateService.cleanupMetaState(generation)); } } @@ -224,7 +224,7 @@ private void writeMetaState(MetaState metaState, List cleanupActions) private Map writeIndicesMetadata(ClusterState newState, ClusterState previousState, List cleanupActions) throws IOException { MetaData previousMetadata = ourState.v2(); - MetaState previousMetastate = ourState.v1(); + Manifest previousMetastate = ourState.v1(); Map previouslyWrittenIndices = previousMetastate.getIndices(); Set relevantIndices = getRelevantIndices(newState, previousState, previouslyWrittenIndices.keySet()); diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java index 122218b965da9..fa3c79f68d680 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java @@ -21,8 +21,8 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.Manifest; import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.cluster.metadata.MetaState; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractComponent; @@ -39,7 +39,7 @@ import java.util.function.Predicate; /** - * Handles writing and loading {@link MetaState}, {@link MetaData} and {@link IndexMetaData} + * Handles writing and loading {@link Manifest}, {@link MetaData} and {@link IndexMetaData} */ public class MetaStateService extends AbstractComponent implements IndexMetaDataWriter { private final NodeEnvironment nodeEnv; @@ -53,29 +53,29 @@ public MetaStateService(Settings settings, NodeEnvironment nodeEnv, NamedXConten /** * Loads the full state, which includes both the global state and all the indices meta data.
- * When loading, manifest file is consulted (represented by {@link MetaState} class), to load proper generations.
+ * When loading, manifest file is consulted (represented by {@link Manifest} class), to load proper generations.
* If there is no manifest file on disk, this method fallbacks to BWC mode, where latest generation of global and indices * metadata is loaded. Please note that currently there is no way to distinguish between manifest file being removed and manifest * file was not yet created. It means that this method always fallbacks to BWC mode, if there is no manifest file. * - * @return tuple of {@link MetaState} and {@link MetaData} with global metadata and indices metadata. If there is no state on disk, + * @return tuple of {@link Manifest} and {@link MetaData} with global metadata and indices metadata. If there is no state on disk, * meta state with globalGeneration -1 and empty meta data is returned. * @throws IOException if some IOException when loading files occurs or there is no metadata referenced by manifest file. */ - Tuple loadFullState() throws IOException { - final MetaState metaState = loadMetaState(); - if (metaState == null) { + Tuple loadFullState() throws IOException { + final Manifest manifest = loadMetaState(); + if (manifest == null) { return loadFullStateBWC(); } final MetaData.Builder metaDataBuilder; - final MetaData globalMetaData = MetaData.FORMAT.loadGeneration(logger, namedXContentRegistry, metaState.getGlobalStateGeneration(), + final MetaData globalMetaData = MetaData.FORMAT.loadGeneration(logger, namedXContentRegistry, manifest.getGlobalStateGeneration(), nodeEnv.nodeDataPaths()); if (globalMetaData != null) { metaDataBuilder = MetaData.builder(globalMetaData); } else { - throw new IOException("failed to find global metadata [generation: " + metaState.getGlobalStateGeneration() + "]"); + throw new IOException("failed to find global metadata [generation: " + manifest.getGlobalStateGeneration() + "]"); } - for (Map.Entry entry : metaState.getIndices().entrySet()) { + for (Map.Entry entry : manifest.getIndices().entrySet()) { Index index = entry.getKey(); long generation = entry.getValue(); final String indexFolderName = index.getUUID(); @@ -88,13 +88,13 @@ Tuple loadFullState() throws IOException { ", generation: " + generation + "]"); } } - return new Tuple<>(metaState, metaDataBuilder.build()); + return new Tuple<>(manifest, metaDataBuilder.build()); } /** * Zen 1 BWC version of loading metadata from disk. See also {@link #loadFullState()} */ - private Tuple loadFullStateBWC() throws IOException { + private Tuple loadFullStateBWC() throws IOException { Map indices = new HashMap<>(); Tuple metaDataAndGeneration = MetaData.FORMAT.loadLatestStateWithGeneration(logger, namedXContentRegistry, nodeEnv.nodeDataPaths()); @@ -125,8 +125,8 @@ private Tuple loadFullStateBWC() throws IOException { logger.debug("[{}] failed to find metadata for existing index location", indexFolderName); } } - MetaState metaState = new MetaState(globalStateGeneration, indices); - return new Tuple<>(metaState, metaDataBuilder.build()); + Manifest manifest = new Manifest(globalStateGeneration, indices); + return new Tuple<>(manifest, metaDataBuilder.build()); } /** @@ -137,8 +137,8 @@ public IndexMetaData loadIndexState(Index index) throws IOException { return IndexMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.indexPaths(index)); } - private MetaState loadMetaState() throws IOException { - return MetaState.FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.nodeDataPaths()); + private Manifest loadMetaState() throws IOException { + return Manifest.FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.nodeDataPaths()); } /** @@ -173,14 +173,14 @@ MetaData loadGlobalState() throws IOException { } /** - * Writes manifest file (represented by {@link MetaState}) to disk. + * Writes manifest file (represented by {@link Manifest}) to disk. * * @throws WriteStateException if exception when writing state occurs. See also {@link WriteStateException#isDirty()} */ - public long writeMetaState(String reason, MetaState metaState) throws WriteStateException { + public long writeManifest(String reason, Manifest manifest) throws WriteStateException { logger.trace("[_meta] writing state, reason [{}]", reason); try { - long generation = MetaState.FORMAT.write(metaState, nodeEnv.nodeDataPaths()); + long generation = Manifest.FORMAT.write(manifest, nodeEnv.nodeDataPaths()); logger.trace("[_meta] state written (generation: {})", generation); return generation; } catch (WriteStateException ex) { @@ -251,19 +251,19 @@ public void cleanupIndex(Index index, long currentGeneration) { * @param currentGeneration current state generation to keep in the directory. */ public void cleanupMetaState(long currentGeneration) { - MetaState.FORMAT.cleanupOldFiles(currentGeneration, nodeEnv.nodeDataPaths()); + Manifest.FORMAT.cleanupOldFiles(currentGeneration, nodeEnv.nodeDataPaths()); } /** * Writes index metadata and updates manifest file accordingly. */ - public void writeIndexAndUpdateMetaState(String reason, IndexMetaData metaData) throws IOException { + public void writeIndexAndUpdateManifest(String reason, IndexMetaData metaData) throws IOException { long generation = writeIndex(reason, metaData); - MetaState metaState = loadMetaState(); - Map indices = new HashMap<>(metaState.getIndices()); + Manifest manifest = loadMetaState(); + Map indices = new HashMap<>(manifest.getIndices()); indices.put(metaData.getIndex(), generation); - metaState = new MetaState(metaState.getGlobalStateGeneration(), indices); - long metaStateGeneration = writeMetaState(reason, metaState); + manifest = new Manifest(manifest.getGlobalStateGeneration(), indices); + long metaStateGeneration = writeManifest(reason, manifest); cleanupIndex(metaData.getIndex(), generation); cleanupMetaState(metaStateGeneration); } @@ -271,11 +271,11 @@ public void writeIndexAndUpdateMetaState(String reason, IndexMetaData metaData) /** * Writes global metadata and updates manifest file accordingly. */ - public void writeGlobalStateAndUpdateMetaState(String reason, MetaData metaData) throws IOException { + public void writeGlobalStateAndUpdateManifest(String reason, MetaData metaData) throws IOException { long generation = writeGlobalState(reason, metaData); - MetaState metaState = loadMetaState(); - metaState = new MetaState(generation, metaState.getIndices()); - long metaStateGeneration = writeMetaState(reason, metaState); + Manifest manifest = loadMetaState(); + manifest = new Manifest(generation, manifest.getIndices()); + long metaStateGeneration = writeManifest(reason, manifest); cleanupGlobalState(generation); cleanupMetaState(metaStateGeneration); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java index feab92adc968e..d49ee658c5de1 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java @@ -361,7 +361,7 @@ public Settings onNodeStopped(String nodeName) throws Exception { .settings(Settings.builder().put(metaData.getSettings()).put("index.foo", true)) .build(); // so evil - metaStateService.writeIndexAndUpdateMetaState("broken metadata", brokenMetaData); + metaStateService.writeIndexAndUpdateManifest("broken metadata", brokenMetaData); } return Settings.EMPTY; } diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java b/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java index da881a0c42637..641dd2ba099d0 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java @@ -405,7 +405,7 @@ public void testRecoverBrokenIndexMetadata() throws Exception { .put("index.analysis.filter.myCollator.type", "icu_collation") ).build(); - metaStateService.writeIndexAndUpdateMetaState("broken metadata", brokenMeta); + metaStateService.writeIndexAndUpdateManifest("broken metadata", brokenMeta); } internalCluster().fullRestart(); // ensureGreen(closedIndex) waits for the index to show up in the metadata @@ -462,7 +462,7 @@ public void testRecoverMissingAnalyzer() throws Exception { for (MetaStateService metaStateService : internalCluster().getInstances(MetaStateService.class)) { IndexMetaData brokenMeta = IndexMetaData.builder(metaData).settings(metaData.getSettings() .filter((s) -> "index.analysis.analyzer.test.tokenizer".equals(s) == false)).build(); - metaStateService.writeIndexAndUpdateMetaState("broken metadata", brokenMeta); + metaStateService.writeIndexAndUpdateManifest("broken metadata", brokenMeta); } internalCluster().fullRestart(); // ensureGreen(closedIndex) waits for the index to show up in the metadata @@ -500,7 +500,7 @@ public void testArchiveBrokenClusterSettings() throws Exception { MetaData brokenMeta = MetaData.builder(metaData).persistentSettings(Settings.builder() .put(metaData.persistentSettings()).put("this.is.unknown", true) .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), "broken").build()).build(); - metaStateService.writeGlobalStateAndUpdateMetaState("broken metadata", brokenMeta); + metaStateService.writeGlobalStateAndUpdateManifest("broken metadata", brokenMeta); } internalCluster().fullRestart(); ensureYellow("test"); // wait for state recovery diff --git a/server/src/test/java/org/elasticsearch/gateway/MetaStateTests.java b/server/src/test/java/org/elasticsearch/gateway/ManifestTests.java similarity index 79% rename from server/src/test/java/org/elasticsearch/gateway/MetaStateTests.java rename to server/src/test/java/org/elasticsearch/gateway/ManifestTests.java index dbbfe08f3d7da..53541ddff8dff 100644 --- a/server/src/test/java/org/elasticsearch/gateway/MetaStateTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/ManifestTests.java @@ -19,7 +19,7 @@ package org.elasticsearch.gateway; -import org.elasticsearch.cluster.metadata.MetaState; +import org.elasticsearch.cluster.metadata.Manifest; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -34,9 +34,9 @@ import static org.hamcrest.Matchers.equalTo; -public class MetaStateTests extends ESTestCase { +public class ManifestTests extends ESTestCase { - private MetaState copyState(MetaState state, boolean introduceErrors){ + private Manifest copyState(Manifest state, boolean introduceErrors){ long generation = state.getGlobalStateGeneration(); Map indices = new HashMap<>(state.getIndices()); if (introduceErrors) { @@ -46,10 +46,10 @@ private MetaState copyState(MetaState state, boolean introduceErrors){ indices.remove(randomFrom(indices.keySet())); } } - return new MetaState(generation, indices); + return new Manifest(generation, indices); } - private MetaState createRandomState() { + private Manifest createRandomState() { long generation = randomNonNegativeLong(); Map indices = new HashMap<>(); for (int i=0; i stateAndData = metaStateService.loadFullState(); - MetaState metaState = stateAndData.v1(); - assertThat(metaState.getGlobalStateGeneration(), equalTo(globalGeneration)); - assertThat(metaState.getIndices(), hasKey(indexMetaData.getIndex())); - assertThat(metaState.getIndices().get(indexMetaData.getIndex()), equalTo(indexGeneration)); + Tuple stateAndData = metaStateService.loadFullState(); + Manifest manifest = stateAndData.v1(); + assertThat(manifest.getGlobalStateGeneration(), equalTo(globalGeneration)); + assertThat(manifest.getIndices(), hasKey(indexMetaData.getIndex())); + assertThat(manifest.getIndices().get(indexMetaData.getIndex()), equalTo(indexGeneration)); MetaData loadedMetaData = stateAndData.v2(); assertThat(loadedMetaData.persistentSettings(), equalTo(metaData.persistentSettings())); @@ -117,11 +117,11 @@ public void testLoadFullStateBWC() throws Exception { public void testLoadEmptyState() throws IOException { try (NodeEnvironment env = newNodeEnvironment()) { MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry()); - Tuple stateAndData = metaStateService.loadFullState(); + Tuple stateAndData = metaStateService.loadFullState(); - MetaState metaState = stateAndData.v1(); - assertThat(metaState.getGlobalStateGeneration(), equalTo(-1L)); - assertThat(metaState.getIndices().entrySet(), empty()); + Manifest manifest = stateAndData.v1(); + assertThat(manifest.getGlobalStateGeneration(), equalTo(-1L)); + assertThat(manifest.getIndices().entrySet(), empty()); MetaData metaData = stateAndData.v2(); MetaData emptyMetaData = MetaData.builder().build(); @@ -142,11 +142,11 @@ public void testLoadFullStateAndUpdate() throws IOException { long globalGeneration = metaStateService.writeGlobalState("first global state write", metaData); long indexGeneration = metaStateService.writeIndex("first index state write", index); - MetaState metaState = new MetaState(globalGeneration, new HashMap() {{ + Manifest manifest = new Manifest(globalGeneration, new HashMap() {{ put(index.getIndex(), indexGeneration); }}); - metaStateService.writeMetaState("first meta state write", metaState); + metaStateService.writeManifest("first meta state write", manifest); MetaData newMetaData = MetaData.builder() .persistentSettings(Settings.builder().put("test1", "value2").build()) @@ -154,25 +154,25 @@ public void testLoadFullStateAndUpdate() throws IOException { .build(); globalGeneration = metaStateService.writeGlobalState("second global state write", newMetaData); - Tuple stateAndData = metaStateService.loadFullState(); - assertThat(stateAndData.v1(), equalTo(metaState)); + Tuple stateAndData = metaStateService.loadFullState(); + assertThat(stateAndData.v1(), equalTo(manifest)); MetaData loadedMetaData = stateAndData.v2(); assertThat(loadedMetaData.persistentSettings(), equalTo(metaData.persistentSettings())); assertThat(loadedMetaData.hasIndex("test1"), equalTo(true)); assertThat(loadedMetaData.index("test1"), equalTo(index)); - metaState = new MetaState(globalGeneration, new HashMap() {{ + manifest = new Manifest(globalGeneration, new HashMap() {{ put(index.getIndex(), indexGeneration); }}); - long metaStateGeneration = metaStateService.writeMetaState("test", metaState); + long metaStateGeneration = metaStateService.writeManifest("test", manifest); metaStateService.cleanupGlobalState(globalGeneration); metaStateService.cleanupIndex(index.getIndex(), indexGeneration); metaStateService.cleanupMetaState(metaStateGeneration); stateAndData = metaStateService.loadFullState(); - assertThat(stateAndData.v1(), equalTo(metaState)); + assertThat(stateAndData.v1(), equalTo(manifest)); loadedMetaData = stateAndData.v2(); assertThat(loadedMetaData.persistentSettings(), equalTo(newMetaData.persistentSettings())); From 25e7c5228acd8331b1a6f5963b81fa1d1bde4023 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Wed, 7 Nov 2018 21:15:31 +0300 Subject: [PATCH 04/57] Change order of ParseField in manifest --- .../main/java/org/elasticsearch/cluster/metadata/Manifest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java index fbf2ae2049734..5e69466622ab0 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java @@ -106,8 +106,8 @@ public Manifest fromXContent(XContentParser parser) throws IOException { * Code below this comment is for XContent manipulation */ - private static final ParseField INDICES_PARSE_FIELD = new ParseField("indices"); private static final ParseField GENERATION_PARSE_FIELD = new ParseField("generation"); + private static final ParseField INDICES_PARSE_FIELD = new ParseField("indices"); @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { From 6440a90fcd963555eb7f4fd12a46201cbc658652 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Wed, 7 Nov 2018 21:19:57 +0300 Subject: [PATCH 05/57] Rename globalGeneration and indexGenerations --- .../cluster/metadata/Manifest.java | 46 +++++++++---------- .../gateway/GatewayMetaState.java | 8 ++-- .../gateway/MetaStateService.java | 12 ++--- .../elasticsearch/gateway/ManifestTests.java | 4 +- .../gateway/MetaStateServiceTests.java | 10 ++-- 5 files changed, 40 insertions(+), 40 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java index 5e69466622ab0..05272b743fe35 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java @@ -40,30 +40,30 @@ * This class represents the manifest file, which is the entry point for reading meta data from disk. * Metadata consists of global metadata and index metadata. * When new version of metadata is written it's assigned some generation long value. - * Global metadata generation could be obtained by calling {@link #getGlobalStateGeneration()}. - * Index metadata generation could be obtained by calling {@link #getIndices()}. + * Global metadata generation could be obtained by calling {@link #getGlobalGeneration()}. + * Index metadata generation could be obtained by calling {@link #getIndexGenerations()}. */ public class Manifest implements ToXContentFragment { - private final long globalStateGeneration; - private final Map indices; + private final long globalGeneration; + private final Map indexGenerations; - public Manifest(long globalStateGeneration, Map indices) { - this.globalStateGeneration = globalStateGeneration; - this.indices = indices; + public Manifest(long globalGeneration, Map indexGenerations) { + this.globalGeneration = globalGeneration; + this.indexGenerations = indexGenerations; } /** * Returns global metadata generation. */ - public long getGlobalStateGeneration() { - return globalStateGeneration; + public long getGlobalGeneration() { + return globalGeneration; } /** * Returns map from {@link Index} to index metadata generation. */ - public Map getIndices() { - return indices; + public Map getIndexGenerations() { + return indexGenerations; } @Override @@ -71,19 +71,19 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Manifest manifest = (Manifest) o; - return globalStateGeneration == manifest.globalStateGeneration && - Objects.equals(indices, manifest.indices); + return globalGeneration == manifest.globalGeneration && + Objects.equals(indexGenerations, manifest.indexGenerations); } @Override public int hashCode() { - return Objects.hash(globalStateGeneration, indices); + return Objects.hash(globalGeneration, indexGenerations); } - private static final String META_STATE_FILE_PREFIX = "meta-"; - private static final ToXContent.Params METASTATE_FORMAT_PARAMS = new ToXContent.MapParams(Collections.singletonMap("binary", "true")); + private static final String MANIFEST_FILE_PREFIX = "manifest-"; + private static final ToXContent.Params MANIFEST_FORMAT_PARAMS = new ToXContent.MapParams(Collections.singletonMap("binary", "true")); - public static final MetaDataStateFormat FORMAT = new MetaDataStateFormat(META_STATE_FILE_PREFIX) { + public static final MetaDataStateFormat FORMAT = new MetaDataStateFormat(MANIFEST_FILE_PREFIX) { @Override protected boolean autoCleanup() { @@ -92,7 +92,7 @@ protected boolean autoCleanup() { @Override public void toXContent(XContentBuilder builder, Manifest state) throws IOException { - state.toXContent(builder, METASTATE_FORMAT_PARAMS); + state.toXContent(builder, MANIFEST_FORMAT_PARAMS); } @Override @@ -107,17 +107,17 @@ public Manifest fromXContent(XContentParser parser) throws IOException { */ private static final ParseField GENERATION_PARSE_FIELD = new ParseField("generation"); - private static final ParseField INDICES_PARSE_FIELD = new ParseField("indices"); + private static final ParseField INDEX_GENERATIONS_PARSE_FIELD = new ParseField("index_generations"); @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(GENERATION_PARSE_FIELD.getPreferredName(), globalStateGeneration); - builder.array(INDICES_PARSE_FIELD.getPreferredName(), indexEntryList().toArray()); + builder.field(GENERATION_PARSE_FIELD.getPreferredName(), globalGeneration); + builder.array(INDEX_GENERATIONS_PARSE_FIELD.getPreferredName(), indexEntryList().toArray()); return builder; } private List indexEntryList() { - return indices.entrySet().stream(). + return indexGenerations.entrySet().stream(). map(entry -> new IndexEntry(entry.getKey(), entry.getValue())). collect(Collectors.toList()); } @@ -138,7 +138,7 @@ private static Map indicies(Object[] generationAndListOfIndexEntrie static { PARSER.declareLong(ConstructingObjectParser.constructorArg(), GENERATION_PARSE_FIELD); - PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), IndexEntry.INDEX_ENTRY_PARSER, INDICES_PARSE_FIELD); + PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), IndexEntry.INDEX_ENTRY_PARSER, INDEX_GENERATIONS_PARSE_FIELD); } public static Manifest fromXContent(XContentParser parser) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index 688a47658b989..f1dcc800e8279 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -108,7 +108,7 @@ private void maybeUpgradeMetaData(MetaDataIndexUpgradeService metaDataIndexUpgra // We finished global state validation and successfully checked all indices for backward compatibility // and found no non-upgradable indices, which means the upgrade can continue. // Now it's safe to overwrite global and index metadata. - long globalStateGeneration = manifest.getGlobalStateGeneration(); + long globalStateGeneration = manifest.getGlobalGeneration(); if (globalStateGeneration != -1) { List cleanupActions = new ArrayList<>(); @@ -225,7 +225,7 @@ private Map writeIndicesMetadata(ClusterState newState, ClusterStat throws IOException { MetaData previousMetadata = ourState.v2(); Manifest previousMetastate = ourState.v1(); - Map previouslyWrittenIndices = previousMetastate.getIndices(); + Map previouslyWrittenIndices = previousMetastate.getIndexGenerations(); Set relevantIndices = getRelevantIndices(newState, previousState, previouslyWrittenIndices.keySet()); Map newIndices = new HashMap<>(); @@ -242,12 +242,12 @@ private Map writeIndicesMetadata(ClusterState newState, ClusterStat } private long writeGlobalState(MetaData newMetaData, List cleanupActions) throws IOException { - if (ourState.v1().getGlobalStateGeneration() == -1 || MetaData.isGlobalStateEquals(ourState.v2(), newMetaData) == false) { + if (ourState.v1().getGlobalGeneration() == -1 || MetaData.isGlobalStateEquals(ourState.v2(), newMetaData) == false) { long generation = metaStateService.writeGlobalState("changed", newMetaData); cleanupActions.add(() -> metaStateService.cleanupGlobalState(generation)); return generation; } - return ourState.v1().getGlobalStateGeneration(); + return ourState.v1().getGlobalGeneration(); } public static Set getRelevantIndices(ClusterState state, ClusterState previousState, Set previouslyWrittenIndices) { diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java index fa3c79f68d680..78f8dfa7f1a77 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java @@ -68,14 +68,14 @@ Tuple loadFullState() throws IOException { return loadFullStateBWC(); } final MetaData.Builder metaDataBuilder; - final MetaData globalMetaData = MetaData.FORMAT.loadGeneration(logger, namedXContentRegistry, manifest.getGlobalStateGeneration(), + final MetaData globalMetaData = MetaData.FORMAT.loadGeneration(logger, namedXContentRegistry, manifest.getGlobalGeneration(), nodeEnv.nodeDataPaths()); if (globalMetaData != null) { metaDataBuilder = MetaData.builder(globalMetaData); } else { - throw new IOException("failed to find global metadata [generation: " + manifest.getGlobalStateGeneration() + "]"); + throw new IOException("failed to find global metadata [generation: " + manifest.getGlobalGeneration() + "]"); } - for (Map.Entry entry : manifest.getIndices().entrySet()) { + for (Map.Entry entry : manifest.getIndexGenerations().entrySet()) { Index index = entry.getKey(); long generation = entry.getValue(); final String indexFolderName = index.getUUID(); @@ -260,9 +260,9 @@ public void cleanupMetaState(long currentGeneration) { public void writeIndexAndUpdateManifest(String reason, IndexMetaData metaData) throws IOException { long generation = writeIndex(reason, metaData); Manifest manifest = loadMetaState(); - Map indices = new HashMap<>(manifest.getIndices()); + Map indices = new HashMap<>(manifest.getIndexGenerations()); indices.put(metaData.getIndex(), generation); - manifest = new Manifest(manifest.getGlobalStateGeneration(), indices); + manifest = new Manifest(manifest.getGlobalGeneration(), indices); long metaStateGeneration = writeManifest(reason, manifest); cleanupIndex(metaData.getIndex(), generation); cleanupMetaState(metaStateGeneration); @@ -274,7 +274,7 @@ public void writeIndexAndUpdateManifest(String reason, IndexMetaData metaData) t public void writeGlobalStateAndUpdateManifest(String reason, MetaData metaData) throws IOException { long generation = writeGlobalState(reason, metaData); Manifest manifest = loadMetaState(); - manifest = new Manifest(generation, manifest.getIndices()); + manifest = new Manifest(generation, manifest.getIndexGenerations()); long metaStateGeneration = writeManifest(reason, manifest); cleanupGlobalState(generation); cleanupMetaState(metaStateGeneration); diff --git a/server/src/test/java/org/elasticsearch/gateway/ManifestTests.java b/server/src/test/java/org/elasticsearch/gateway/ManifestTests.java index 53541ddff8dff..2e26f64844146 100644 --- a/server/src/test/java/org/elasticsearch/gateway/ManifestTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/ManifestTests.java @@ -37,8 +37,8 @@ public class ManifestTests extends ESTestCase { private Manifest copyState(Manifest state, boolean introduceErrors){ - long generation = state.getGlobalStateGeneration(); - Map indices = new HashMap<>(state.getIndices()); + long generation = state.getGlobalGeneration(); + Map indices = new HashMap<>(state.getIndexGenerations()); if (introduceErrors) { if (randomBoolean()){ generation = generation + 1; diff --git a/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java b/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java index a29ca753013ca..df1fbc8a7ebc2 100644 --- a/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java @@ -103,9 +103,9 @@ public void testLoadFullStateBWC() throws Exception { Tuple stateAndData = metaStateService.loadFullState(); Manifest manifest = stateAndData.v1(); - assertThat(manifest.getGlobalStateGeneration(), equalTo(globalGeneration)); - assertThat(manifest.getIndices(), hasKey(indexMetaData.getIndex())); - assertThat(manifest.getIndices().get(indexMetaData.getIndex()), equalTo(indexGeneration)); + assertThat(manifest.getGlobalGeneration(), equalTo(globalGeneration)); + assertThat(manifest.getIndexGenerations(), hasKey(indexMetaData.getIndex())); + assertThat(manifest.getIndexGenerations().get(indexMetaData.getIndex()), equalTo(indexGeneration)); MetaData loadedMetaData = stateAndData.v2(); assertThat(loadedMetaData.persistentSettings(), equalTo(metaData.persistentSettings())); @@ -120,8 +120,8 @@ public void testLoadEmptyState() throws IOException { Tuple stateAndData = metaStateService.loadFullState(); Manifest manifest = stateAndData.v1(); - assertThat(manifest.getGlobalStateGeneration(), equalTo(-1L)); - assertThat(manifest.getIndices().entrySet(), empty()); + assertThat(manifest.getGlobalGeneration(), equalTo(-1L)); + assertThat(manifest.getIndexGenerations().entrySet(), empty()); MetaData metaData = stateAndData.v2(); MetaData emptyMetaData = MetaData.builder().build(); From aba2e40263bbe397edcd0565c2e2468d4e0b5d05 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Wed, 7 Nov 2018 21:28:23 +0300 Subject: [PATCH 06/57] writeAndCleanup method --- .../cluster/metadata/IndexMetaData.java | 5 ---- .../cluster/metadata/Manifest.java | 5 ---- .../cluster/metadata/MetaData.java | 5 ---- .../elasticsearch/env/NodeEnvironment.java | 2 +- .../gateway/MetaDataStateFormat.java | 25 ++++++++++--------- .../elasticsearch/index/shard/IndexShard.java | 2 +- .../RemoveCorruptedShardDataCommand.java | 2 +- .../gateway/MetaDataStateFormatTests.java | 20 +++++++-------- .../index/shard/IndexShardTests.java | 2 +- .../RemoveCorruptedShardDataCommandTests.java | 2 +- .../index/shard/ShardPathTests.java | 12 ++++++--- 11 files changed, 36 insertions(+), 46 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 355c4fa4f7f8d..b1b092e008679 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -1408,11 +1408,6 @@ public static Settings addHumanReadableSettings(Settings settings) { */ public static final MetaDataStateFormat FORMAT = new MetaDataStateFormat(INDEX_STATE_FILE_PREFIX) { - @Override - protected boolean autoCleanup() { - return false; - } - @Override public void toXContent(XContentBuilder builder, IndexMetaData state) throws IOException { Builder.toXContent(state, builder, FORMAT_PARAMS); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java index 05272b743fe35..41746aa9b418e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java @@ -85,11 +85,6 @@ public int hashCode() { public static final MetaDataStateFormat FORMAT = new MetaDataStateFormat(MANIFEST_FILE_PREFIX) { - @Override - protected boolean autoCleanup() { - return false; - } - @Override public void toXContent(XContentBuilder builder, Manifest state) throws IOException { state.toXContent(builder, MANIFEST_FORMAT_PARAMS); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index a8577b83776e9..acd28a55604d3 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -1321,11 +1321,6 @@ public static MetaData fromXContent(XContentParser parser) throws IOException { */ public static final MetaDataStateFormat FORMAT = new MetaDataStateFormat(GLOBAL_STATE_FILE_PREFIX) { - @Override - protected boolean autoCleanup() { - return false; - } - @Override public void toXContent(XContentBuilder builder, MetaData state) throws IOException { Builder.toXContent(state, builder, FORMAT_PARAMS); diff --git a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java index ff8baaabb443c..f07e5da437d67 100644 --- a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java +++ b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java @@ -390,7 +390,7 @@ private static NodeMetaData loadOrCreateNodeMetaData(Settings settings, Logger l metaData = new NodeMetaData(generateNodeId(settings)); } // we write again to make sure all paths have the latest state file - NodeMetaData.FORMAT.write(metaData, paths); + NodeMetaData.FORMAT.writeAndCleanup(metaData, paths); return metaData; } diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java b/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java index 2474e570bb60a..b536a7605926f 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java @@ -176,6 +176,17 @@ private static void performStateDirectoriesFsync(List> st } } + /** + * Writes the given state to the given directories and performs cleanup. + * See also {@link #write(Object, Path...)} and {@link #cleanupOldFiles(long, Path[])}. + */ + public final long writeAndCleanup(final T state, final Path... locations) throws WriteStateException { + long currentGeneration = write(state, locations); + cleanupOldFiles(currentGeneration,locations); + return currentGeneration; + } + + /** * Writes the given state to the given directories. The state is written to a * state directory ({@value #STATE_DIR_NAME}) underneath each of the given file locations and is created if it @@ -183,10 +194,10 @@ private static void performStateDirectoriesFsync(List> st * it's target filename of the pattern {@code {prefix}{version}.st}. * If this method returns without exception there is a guarantee that state is persisted to the disk and loadLatestState will return * it.
- * If {@link #autoCleanup()} returns false, this method does not perform cleanup of old state files, - * because one write could be a part of larger transaction. + * This method does not perform cleanup of old state files. * If this write succeeds, but some further write fails, you may want to rollback the transaction and keep old file around. * After transaction is finished use {@link #cleanupOldFiles(long, Path[])} for the clean-up. + * If this write is not a part of bigger transaction, consider using {@link #writeAndCleanup(Object, Path...)} method instead. * * @param state the state object to write * @param locations the locations where the state should be written to. @@ -234,10 +245,6 @@ public final long write(final T state, final Path... locations) throws WriteStat } } - if (autoCleanup()) { - cleanupOldFiles(maxStateId, locations); - } - return maxStateId; } @@ -291,12 +298,6 @@ protected Directory newDirectory(Path dir) throws IOException { return new SimpleFSDirectory(dir); } - /** - * Whether to perform autoCleanup of old state files after successful {@link #write(Object, Path...)}. - */ - protected boolean autoCleanup() { - return true; - } /** * Clean ups all state files not matching passed generation. diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 17756630517d2..580f75c4c08e8 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -2243,7 +2243,7 @@ private static void persistMetadata( logger.trace("{} writing shard state, reason [{}]", shardId, writeReason); final ShardStateMetaData newShardStateMetadata = new ShardStateMetaData(newRouting.primary(), indexSettings.getUUID(), newRouting.allocationId()); - ShardStateMetaData.FORMAT.write(newShardStateMetadata, shardPath.getShardStatePath()); + ShardStateMetaData.FORMAT.writeAndCleanup(newShardStateMetadata, shardPath.getShardStatePath()); } else { logger.trace("{} skip writing shard state, has been written before", shardId); } diff --git a/server/src/main/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommand.java b/server/src/main/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommand.java index 9535108cad3c0..7242198633be2 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommand.java +++ b/server/src/main/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommand.java @@ -461,7 +461,7 @@ protected void newAllocationId(Environment environment, ShardPath shardPath, Ter final ShardStateMetaData newShardStateMetaData = new ShardStateMetaData(shardStateMetaData.primary, shardStateMetaData.indexUUID, newAllocationId); - ShardStateMetaData.FORMAT.write(newShardStateMetaData, shardStatePath); + ShardStateMetaData.FORMAT.writeAndCleanup(newShardStateMetaData, shardStatePath); terminal.println(""); terminal.println("You should run the following command to allocate this shard:"); diff --git a/server/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java b/server/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java index 8c97c5cfa3313..854c12087a483 100644 --- a/server/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java @@ -103,7 +103,7 @@ public void testReadWriteState() throws IOException { Format format = new Format("foo-"); DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean()); - format.write(state, dirs); + format.writeAndCleanup(state, dirs); for (Path file : dirs) { Path[] list = content("*", file); assertEquals(list.length, 1); @@ -118,7 +118,7 @@ public void testReadWriteState() throws IOException { } DummyState state2 = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean()); - format.write(state2, dirs); + format.writeAndCleanup(state2, dirs); for (Path file : dirs) { Path[] list = content("*", file); @@ -145,7 +145,7 @@ public void testVersionMismatch() throws IOException { Format format = new Format("foo-"); DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean()); - format.write(state, dirs); + format.writeAndCleanup(state, dirs); for (Path file : dirs) { Path[] list = content("*", file); assertEquals(list.length, 1); @@ -169,7 +169,7 @@ public void testCorruption() throws IOException { Format format = new Format("foo-"); DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean()); - format.write(state, dirs); + format.writeAndCleanup(state, dirs); for (Path file : dirs) { Path[] list = content("*", file); assertEquals(list.length, 1); @@ -247,7 +247,7 @@ public void testLoadState() throws IOException { dirs[i] = createTempDir(); Files.createDirectories(dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME)); for (int j = 0; j < numStates; j++) { - format.write(meta.get(j), dirs[i]); + format.writeAndCleanup(meta.get(j), dirs[i]); if (randomBoolean() && (j < numStates - 1 || dirs.length > 0 && i != 0)) { // corrupt a file that we do not necessarily // need here.... Path file = dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-" + j + ".st"); @@ -298,7 +298,7 @@ private DummyState writeAndReadStateSuccessfully(Format format, Path... paths) t format.noFailures(); DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 100), randomInt(), randomLong(), randomDouble(), randomBoolean()); - format.write(state, paths); + format.writeAndCleanup(state, paths); assertEquals(state, format.loadLatestState(logger, NamedXContentRegistry.EMPTY, paths)); ensureOnlyOneStateFile(paths); return state; @@ -323,7 +323,7 @@ public void testFailWriteAndReadPreviousState() throws IOException { Format.FAIL_FSYNC_TMP_FILE, Format.FAIL_RENAME_TMP_FILE); DummyState newState = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 100), randomInt(), randomLong(), randomDouble(), randomBoolean()); - WriteStateException ex = expectThrows(WriteStateException.class, () -> format.write(newState, path)); + WriteStateException ex = expectThrows(WriteStateException.class, () -> format.writeAndCleanup(newState, path)); assertFalse(ex.isDirty()); format.noFailures(); @@ -346,7 +346,7 @@ public void testFailWriteAndReadAnyState() throws IOException { DummyState newState = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 100), randomInt(), randomLong(), randomDouble(), randomBoolean()); possibleStates.add(newState); - WriteStateException ex = expectThrows(WriteStateException.class, () -> format.write(newState, path)); + WriteStateException ex = expectThrows(WriteStateException.class, () -> format.writeAndCleanup(newState, path)); assertTrue(ex.isDirty()); format.noFailures(); @@ -369,7 +369,7 @@ public void testFailCopyTmpFileToExtraLocation() throws IOException { format.failOnMethods(Format.FAIL_OPEN_STATE_FILE_WHEN_COPYING); DummyState newState = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 100), randomInt(), randomLong(), randomDouble(), randomBoolean()); - WriteStateException ex = expectThrows(WriteStateException.class, () -> format.write(newState, paths)); + WriteStateException ex = expectThrows(WriteStateException.class, () -> format.writeAndCleanup(newState, paths)); assertFalse(ex.isDirty()); format.noFailures(); @@ -395,7 +395,7 @@ public void testFailRandomlyAndReadAnyState() throws IOException { DummyState newState = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 100), randomInt(), randomLong(), randomDouble(), randomBoolean()); try { - format.write(newState, paths); + format.writeAndCleanup(newState, paths); possibleStates.clear(); possibleStates.add(newState); } catch (WriteStateException e) { diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 487ac7e0694ef..e365b5ec742fe 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -194,7 +194,7 @@ public static ShardStateMetaData load(Logger logger, Path... shardPaths) throws public static void write(ShardStateMetaData shardStateMetaData, Path... shardPaths) throws IOException { - ShardStateMetaData.FORMAT.write(shardStateMetaData, shardPaths); + ShardStateMetaData.FORMAT.writeAndCleanup(shardStateMetaData, shardPaths); } public static Engine getEngineFromShard(IndexShard shard) { diff --git a/server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandTests.java b/server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandTests.java index 6e34bb03860c5..93e0515c1cde7 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandTests.java @@ -401,7 +401,7 @@ private void writeIndexState() throws IOException { // create _state of IndexMetaData try(NodeEnvironment nodeEnvironment = new NodeEnvironment(environment.settings(), environment)) { final Path[] paths = nodeEnvironment.indexPaths(indexMetaData.getIndex()); - IndexMetaData.FORMAT.write(indexMetaData, paths); + IndexMetaData.FORMAT.writeAndCleanup(indexMetaData, paths); logger.info("--> index metadata persisted to {} ", Arrays.toString(paths)); } } diff --git a/server/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java b/server/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java index fda2f8ef7d039..dd247d8d7fc97 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java @@ -43,7 +43,8 @@ public void testLoadShardPath() throws IOException { ShardId shardId = new ShardId("foo", "0xDEADBEEF", 0); Path[] paths = env.availableShardPaths(shardId); Path path = randomFrom(paths); - ShardStateMetaData.FORMAT.write(new ShardStateMetaData(true, "0xDEADBEEF", AllocationId.newInitializing()), path); + ShardStateMetaData.FORMAT.writeAndCleanup( + new ShardStateMetaData(true, "0xDEADBEEF", AllocationId.newInitializing()), path); ShardPath shardPath = ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.getIndex(), settings)); assertEquals(path, shardPath.getDataPath()); assertEquals("0xDEADBEEF", shardPath.getShardId().getIndex().getUUID()); @@ -62,7 +63,8 @@ public void testFailLoadShardPathOnMultiState() throws IOException { ShardId shardId = new ShardId("foo", indexUUID, 0); Path[] paths = env.availableShardPaths(shardId); assumeTrue("This test tests multi data.path but we only got one", paths.length > 1); - ShardStateMetaData.FORMAT.write(new ShardStateMetaData(true, indexUUID, AllocationId.newInitializing()), paths); + ShardStateMetaData.FORMAT.writeAndCleanup( + new ShardStateMetaData(true, indexUUID, AllocationId.newInitializing()), paths); Exception e = expectThrows(IllegalStateException.class, () -> ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.getIndex(), settings))); assertThat(e.getMessage(), containsString("more than one shard state found")); @@ -77,7 +79,8 @@ public void testFailLoadShardPathIndexUUIDMissmatch() throws IOException { ShardId shardId = new ShardId("foo", "foobar", 0); Path[] paths = env.availableShardPaths(shardId); Path path = randomFrom(paths); - ShardStateMetaData.FORMAT.write(new ShardStateMetaData(true, "0xDEADBEEF", AllocationId.newInitializing()), path); + ShardStateMetaData.FORMAT.writeAndCleanup( + new ShardStateMetaData(true, "0xDEADBEEF", AllocationId.newInitializing()), path); Exception e = expectThrows(IllegalStateException.class, () -> ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.getIndex(), settings))); assertThat(e.getMessage(), containsString("expected: foobar on shard path")); @@ -124,7 +127,8 @@ public void testGetRootPaths() throws IOException { ShardId shardId = new ShardId("foo", indexUUID, 0); Path[] paths = env.availableShardPaths(shardId); Path path = randomFrom(paths); - ShardStateMetaData.FORMAT.write(new ShardStateMetaData(true, indexUUID, AllocationId.newInitializing()), path); + ShardStateMetaData.FORMAT.writeAndCleanup( + new ShardStateMetaData(true, indexUUID, AllocationId.newInitializing()), path); ShardPath shardPath = ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.getIndex(), indexSettings, nodeSettings)); boolean found = false; From b6ca5d9f814dc56dca99fbd4556b8a6cd8679b71 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Wed, 7 Nov 2018 21:29:09 +0300 Subject: [PATCH 07/57] s/of/if --- .../java/org/elasticsearch/gateway/MetaDataStateFormat.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java b/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java index b536a7605926f..829c48a54766e 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java @@ -411,7 +411,7 @@ public T loadGeneration(Logger logger, NamedXContentRegistry namedXContentRegist * * @param logger a logger instance. * @param dataLocations the data-locations to try. - * @return tuple of the latest state and generation. (-1, null) of no state is found. + * @return tuple of the latest state and generation. (-1, null) if no state is found. */ public Tuple loadLatestStateWithGeneration(Logger logger, NamedXContentRegistry namedXContentRegistry, Path... dataLocations) throws IOException { From 9d482adc9fdf6c15de4d35a9d83f27c378fc7ddc Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Wed, 7 Nov 2018 21:30:38 +0300 Subject: [PATCH 08/57] s/state id/generation id --- .../java/org/elasticsearch/gateway/MetaDataStateFormat.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java b/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java index 829c48a54766e..7f9eacf89b804 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java @@ -216,7 +216,7 @@ public final long write(final T state, final Path... locations) throws WriteStat try { maxStateId = findMaxStateId(prefix, locations) + 1; } catch (Exception e) { - throw new WriteStateException(false, "exception during looking up max state id", e); + throw new WriteStateException(false, "exception during looking up max generation id", e); } assert maxStateId >= 0 : "maxStateId must be positive but was: [" + maxStateId + "]"; @@ -387,7 +387,7 @@ public T loadGeneration(Logger logger, NamedXContentRegistry namedXContentRegist for (Path stateFile : stateFiles) { try { T state = read(namedXContentRegistry, stateFile); - logger.trace("state id [{}] read from [{}]", generation, stateFile.getFileName()); + logger.trace("generation id [{}] read from [{}]", generation, stateFile.getFileName()); return state; } catch (Exception e) { exceptions.add(new IOException("failed to read " + stateFile.toAbsolutePath(), e)); @@ -419,7 +419,7 @@ public Tuple loadLatestStateWithGeneration(Logger logger, NamedXContent T state = loadGeneration(logger, namedXContentRegistry, generation, dataLocations); if (generation > -1 && state == null) { - throw new IllegalStateException("unable to find state files with state id " + generation + + throw new IllegalStateException("unable to find state files with generation id " + generation + " returned by findMaxStateId function, in data folders [" + Arrays.stream(dataLocations).map(Path::toAbsolutePath). map(Object::toString).collect(Collectors.joining(", ")) + From 23c3ca6a985f627d4cc1a8863a5cedeb7de2e55b Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Wed, 7 Nov 2018 21:31:43 +0300 Subject: [PATCH 09/57] s/zen 1/manifest-less --- .../main/java/org/elasticsearch/gateway/MetaStateService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java index 78f8dfa7f1a77..19b6cce1e7aa6 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java @@ -92,7 +92,7 @@ Tuple loadFullState() throws IOException { } /** - * Zen 1 BWC version of loading metadata from disk. See also {@link #loadFullState()} + * "Manifest-less" BWC version of loading metadata from disk. See also {@link #loadFullState()} */ private Tuple loadFullStateBWC() throws IOException { Map indices = new HashMap<>(); From 3e090b432c6f9f5082b8a88d3361a87c2c938c00 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Wed, 7 Nov 2018 21:35:04 +0300 Subject: [PATCH 10/57] ManifestTest: move, reformat, use eq&hash utils, enrich mutation --- .../metadata}/ManifestTests.java | 67 +++++++++++-------- 1 file changed, 40 insertions(+), 27 deletions(-) rename server/src/test/java/org/elasticsearch/{gateway => cluster/metadata}/ManifestTests.java (56%) diff --git a/server/src/test/java/org/elasticsearch/gateway/ManifestTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ManifestTests.java similarity index 56% rename from server/src/test/java/org/elasticsearch/gateway/ManifestTests.java rename to server/src/test/java/org/elasticsearch/cluster/metadata/ManifestTests.java index 2e26f64844146..6c2bdbe8ea0cd 100644 --- a/server/src/test/java/org/elasticsearch/gateway/ManifestTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ManifestTests.java @@ -17,11 +17,11 @@ * under the License. */ -package org.elasticsearch.gateway; +package org.elasticsearch.cluster.metadata; -import org.elasticsearch.cluster.metadata.Manifest; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.json.JsonXContent; @@ -32,50 +32,63 @@ import java.util.HashMap; import java.util.Map; +import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; import static org.hamcrest.Matchers.equalTo; public class ManifestTests extends ESTestCase { - private Manifest copyState(Manifest state, boolean introduceErrors){ + private Manifest copyState(Manifest state, boolean introduceErrors) { long generation = state.getGlobalGeneration(); Map indices = new HashMap<>(state.getIndexGenerations()); if (introduceErrors) { - if (randomBoolean()){ - generation = generation + 1; - } else { - indices.remove(randomFrom(indices.keySet())); + switch (randomInt(3)) { + case 0: { + generation = generation + 1; + break; + } + case 1: { + indices.remove(randomFrom(indices.keySet())); + break; + } + case 2: { + Tuple indexEntry = randomIndexEntry(); + indices.put(indexEntry.v1(), indexEntry.v2()); + break; + } + case 3: { + Index index = randomFrom(indices.keySet()); + indices.compute(index, (i, g) -> g + 1); + break; + } } } return new Manifest(generation, indices); } - private Manifest createRandomState() { + private Tuple randomIndexEntry() { + final String name = randomAlphaOfLengthBetween(4, 15); + final String uuid = UUIDs.randomBase64UUID(); + final Index index = new Index(name, uuid); + final long indexGeneration = randomNonNegativeLong(); + return Tuple.tuple(index, indexGeneration); + } + + private Manifest randomManifest() { long generation = randomNonNegativeLong(); Map indices = new HashMap<>(); - for (int i=0; i indexEntry = randomIndexEntry(); + indices.put(indexEntry.v1(), indexEntry.v2()); } - return new Manifest(generation, indices); - } - - public void testEquals(){ - Manifest state = createRandomState(); - Manifest copy = copyState(state, false); - assertEquals(state, copy); + return new Manifest(generation, indices); } - public void testNonEquals(){ - Manifest state = createRandomState(); - Manifest copy = copyState(state, true); - assertNotEquals(state, copy); + public void testEqualsAndHashCode() { + checkEqualsAndHashCode(randomManifest(), org -> copyState(org, false), org -> copyState(org, true)); } public void testXContent() throws IOException { - Manifest state = createRandomState(); + Manifest state = randomManifest(); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); @@ -86,4 +99,4 @@ public void testXContent() throws IOException { assertThat(Manifest.fromXContent(parser), equalTo(state)); } } -} +} \ No newline at end of file From 988ed6d80613716ec30d78075e88b9c46d271f3e Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Wed, 7 Nov 2018 21:35:54 +0300 Subject: [PATCH 11/57] fileToCorrupt = file --- .../org/elasticsearch/gateway/MetaDataStateFormatTests.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java b/server/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java index 854c12087a483..a7f24cdba3a52 100644 --- a/server/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java @@ -192,8 +192,7 @@ public void testCorruption() throws IOException { } } - public static void corruptFile(Path file, Logger logger) throws IOException { - Path fileToCorrupt = file; + public static void corruptFile(Path fileToCorrupt, Logger logger) throws IOException { try (SimpleFSDirectory dir = new SimpleFSDirectory(fileToCorrupt.getParent())) { long checksumBeforeCorruption; try (IndexInput input = dir.openInput(fileToCorrupt.getFileName().toString(), IOContext.DEFAULT)) { From 87f6ae9ff9d596bed8ab59aa4077e18dbf9a698b Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Wed, 7 Nov 2018 21:51:48 +0300 Subject: [PATCH 12/57] Fix tests: FullClusterRestart callback --- .../admin/indices/create/CreateIndexIT.java | 28 +++++++------ .../gateway/GatewayIndexStateIT.java | 41 ++++++++++++------- .../gateway/QuorumGatewayIT.java | 3 +- .../gateway/RecoveryFromGatewayIT.java | 4 +- .../test/InternalTestCluster.java | 19 ++++++--- 5 files changed, 61 insertions(+), 34 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java index d49ee658c5de1..e88168971f37d 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java @@ -45,6 +45,7 @@ import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; @@ -350,22 +351,25 @@ public void testIndexWithUnknownSetting() throws Exception { } final IndexMetaData metaData = state.getMetaData().index("test"); - internalCluster().fullRestart(new InternalTestCluster.RestartCallback() { + internalCluster().fullRestart(new InternalTestCluster.FullRestartCallback() { + @Override - public Settings onNodeStopped(String nodeName) throws Exception { - if (dataOrMasterNodeNames.contains(nodeName)) { - final MetaStateService metaStateService = internalCluster().getInstance(MetaStateService.class, nodeName); - final IndexMetaData brokenMetaData = - IndexMetaData - .builder(metaData) - .settings(Settings.builder().put(metaData.getSettings()).put("index.foo", true)) - .build(); - // so evil - metaStateService.writeIndexAndUpdateManifest("broken metadata", brokenMetaData); + public void onAllNodesStopped(List nodeNames) throws Exception { + for (String nodeName : nodeNames) { + if (dataOrMasterNodeNames.contains(nodeName)) { + final MetaStateService metaStateService = internalCluster().getInstance(MetaStateService.class, nodeName); + final IndexMetaData brokenMetaData = + IndexMetaData + .builder(metaData) + .settings(Settings.builder().put(metaData.getSettings()).put("index.foo", true)) + .build(); + // so evil + metaStateService.writeIndexAndUpdateManifest("broken metadata", brokenMetaData); + } } - return Settings.EMPTY; } }); + ensureGreen(metaData.getIndex().getName()); // we have to wait for the index to show up in the metadata or we will fail in a race final ClusterState stateAfterRestart = client().admin().cluster().prepareState().get().getState(); diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java b/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java index 641dd2ba099d0..704231907bef5 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java @@ -48,6 +48,7 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; +import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.InternalTestCluster.RestartCallback; import java.io.IOException; @@ -291,7 +292,7 @@ public void testDanglingIndices() throws Exception { assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(true)); logger.info("--> restarting the nodes"); - internalCluster().fullRestart(new RestartCallback() { + internalCluster().fullRestart(new InternalTestCluster.FullRestartCallback() { @Override public boolean clearData(String nodeName) { return node_1.equals(nodeName); @@ -459,12 +460,18 @@ public void testRecoverMissingAnalyzer() throws Exception { } ClusterState state = client().admin().cluster().prepareState().get().getState(); IndexMetaData metaData = state.getMetaData().index("test"); - for (MetaStateService metaStateService : internalCluster().getInstances(MetaStateService.class)) { - IndexMetaData brokenMeta = IndexMetaData.builder(metaData).settings(metaData.getSettings() - .filter((s) -> "index.analysis.analyzer.test.tokenizer".equals(s) == false)).build(); - metaStateService.writeIndexAndUpdateManifest("broken metadata", brokenMeta); - } - internalCluster().fullRestart(); + + internalCluster().fullRestart(new InternalTestCluster.FullRestartCallback(){ + @Override + public void onAllNodesStopped(List nodeNames) throws Exception { + for (MetaStateService metaStateService : internalCluster().getInstances(MetaStateService.class)) { + IndexMetaData brokenMeta = IndexMetaData.builder(metaData).settings(metaData.getSettings() + .filter((s) -> "index.analysis.analyzer.test.tokenizer".equals(s) == false)).build(); + metaStateService.writeIndexAndUpdateManifest("broken metadata", brokenMeta); + } + } + }); + // ensureGreen(closedIndex) waits for the index to show up in the metadata // this is crucial otherwise the state call below might not contain the index yet ensureGreen(metaData.getIndex().getName()); @@ -496,13 +503,19 @@ public void testArchiveBrokenClusterSettings() throws Exception { } ClusterState state = client().admin().cluster().prepareState().get().getState(); MetaData metaData = state.getMetaData(); - for (MetaStateService metaStateService : internalCluster().getInstances(MetaStateService.class)) { - MetaData brokenMeta = MetaData.builder(metaData).persistentSettings(Settings.builder() - .put(metaData.persistentSettings()).put("this.is.unknown", true) - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), "broken").build()).build(); - metaStateService.writeGlobalStateAndUpdateManifest("broken metadata", brokenMeta); - } - internalCluster().fullRestart(); + + internalCluster().fullRestart(new InternalTestCluster.FullRestartCallback(){ + @Override + public void onAllNodesStopped(List nodeNames) throws Exception { + for (MetaStateService metaStateService : internalCluster().getInstances(MetaStateService.class)) { + MetaData brokenMeta = MetaData.builder(metaData).persistentSettings(Settings.builder() + .put(metaData.persistentSettings()).put("this.is.unknown", true) + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), "broken").build()).build(); + metaStateService.writeGlobalStateAndUpdateManifest("broken metadata", brokenMeta); + } + } + }); + ensureYellow("test"); // wait for state recovery state = client().admin().cluster().prepareState().get().getState(); assertEquals("true", state.metaData().persistentSettings().get("archived.this.is.unknown")); diff --git a/server/src/test/java/org/elasticsearch/gateway/QuorumGatewayIT.java b/server/src/test/java/org/elasticsearch/gateway/QuorumGatewayIT.java index b16e2e2f6c505..cbd2b656875e4 100644 --- a/server/src/test/java/org/elasticsearch/gateway/QuorumGatewayIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/QuorumGatewayIT.java @@ -26,6 +26,7 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; +import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.InternalTestCluster.RestartCallback; import java.util.concurrent.TimeUnit; @@ -64,7 +65,7 @@ public void testQuorumRecovery() throws Exception { assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).get(), 2L); } logger.info("--> restart all nodes"); - internalCluster().fullRestart(new RestartCallback() { + internalCluster().fullRestart(new InternalTestCluster.FullRestartCallback() { @Override public Settings onNodeStopped(String nodeName) throws Exception { return null; diff --git a/server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java b/server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java index a8f2cfab2b79b..43ba81f91f6d3 100644 --- a/server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java @@ -300,7 +300,7 @@ public void testTwoNodeFirstNodeCleared() throws Exception { Map primaryTerms = assertAndCapturePrimaryTerms(null); - internalCluster().fullRestart(new RestartCallback() { + internalCluster().fullRestart(new InternalTestCluster.FullRestartCallback() { @Override public Settings onNodeStopped(String nodeName) throws Exception { return Settings.builder().put("gateway.recover_after_nodes", 2).build(); @@ -549,7 +549,7 @@ public void testStartedShardFoundIfStateNotYetProcessed() throws Exception { final boolean corrupt = randomBoolean(); - internalCluster().fullRestart(new RestartCallback() { + internalCluster().fullRestart(new InternalTestCluster.FullRestartCallback() { @Override public Settings onNodeStopped(String nodeName) throws Exception { // make sure state is not recovered diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index 718e4928d6cd4..0eec82e8d00a9 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -1687,7 +1687,7 @@ public Settings onNodeStopped(String node) { * Restarts all nodes in the cluster. It first stops all nodes and then restarts all the nodes again. */ public void fullRestart() throws Exception { - fullRestart(EMPTY_CALLBACK); + fullRestart(new FullRestartCallback()); } /** @@ -1731,7 +1731,7 @@ private void restartNode(NodeAndClient nodeAndClient, RestartCallback callback) /** * Restarts all nodes in the cluster. It first stops all nodes and then restarts all the nodes again. */ - public synchronized void fullRestart(RestartCallback callback) throws Exception { + public synchronized void fullRestart(FullRestartCallback callback) throws Exception { int numNodesRestarted = 0; Map, List> nodesByRoles = new HashMap<>(); Set[] rolesOrderedByOriginalStartupOrder = new Set[nextNodeId.get()]; @@ -1749,6 +1749,8 @@ public synchronized void fullRestart(RestartCallback callback) throws Exception nodesByRoles.computeIfAbsent(discoveryNode.getRoles(), k -> new ArrayList<>()).add(nodeAndClient); } + callback.onAllNodesStopped(nodes.values().stream().map(nodeAndClient -> nodeAndClient.name).collect(Collectors.toList())); + assert nodesByRoles.values().stream().collect(Collectors.summingInt(List::size)) == nodes.size(); // randomize start up order, but making sure that: @@ -2203,11 +2205,18 @@ public boolean test(Settings settings) { } } + /** + * A class that is called during {@link #fullRestart(FullRestartCallback)} + * to execute actions at certain stages of the restart. + */ + public static class FullRestartCallback extends RestartCallback { + public void onAllNodesStopped(List nodeNames) throws Exception { + } + } /** - * An abstract class that is called during {@link #rollingRestart(InternalTestCluster.RestartCallback)} - * and / or {@link #fullRestart(InternalTestCluster.RestartCallback)} to execute actions at certain - * stages of the restart. + * A class that is called during {@link #rollingRestart(InternalTestCluster.RestartCallback)} + * to execute actions at certain stages of the restart. */ public static class RestartCallback { From b1eb33eae792e3c4551c054e632b3b4ca4ee684d Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Wed, 7 Nov 2018 22:06:18 +0300 Subject: [PATCH 13/57] indices typo fix --- .../java/org/elasticsearch/cluster/metadata/Manifest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java index 41746aa9b418e..63b1e2b9be796 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java @@ -121,7 +121,7 @@ private static long generation(Object[] generationAndListOfIndexEntries) { return (Long) generationAndListOfIndexEntries[0]; } - private static Map indicies(Object[] generationAndListOfIndexEntries) { + private static Map indices(Object[] generationAndListOfIndexEntries) { List listOfIndices = (List) generationAndListOfIndexEntries[1]; return listOfIndices.stream().collect(Collectors.toMap(IndexEntry::getIndex, IndexEntry::getGeneration)); } @@ -129,7 +129,7 @@ private static Map indicies(Object[] generationAndListOfIndexEntrie private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "manifest", generationAndListOfIndexEntries -> - new Manifest(generation(generationAndListOfIndexEntries), indicies(generationAndListOfIndexEntries))); + new Manifest(generation(generationAndListOfIndexEntries), indices(generationAndListOfIndexEntries))); static { PARSER.declareLong(ConstructingObjectParser.constructorArg(), GENERATION_PARSE_FIELD); From f5cd21f3b409fcd0dc084d22dfe77b068b37b9aa Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Wed, 7 Nov 2018 22:06:34 +0300 Subject: [PATCH 14/57] MetaStateServiceTests refactoring --- .../gateway/MetaStateServiceTests.java | 241 +++++++++--------- 1 file changed, 119 insertions(+), 122 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java b/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java index df1fbc8a7ebc2..699fd728606ee 100644 --- a/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.Manifest; import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.NodeEnvironment; @@ -37,147 +38,143 @@ import static org.hamcrest.Matchers.nullValue; public class MetaStateServiceTests extends ESTestCase { - private static Settings indexSettings = Settings.builder() - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) - .build(); - public void testWriteLoadIndex() throws Exception { - try (NodeEnvironment env = newNodeEnvironment()) { - MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry()); + private NodeEnvironment env; + private MetaStateService metaStateService; + + @Override + public void setUp() throws Exception { + super.setUp(); + env = newNodeEnvironment(); + metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry()); + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + env.close(); + } - IndexMetaData index = IndexMetaData.builder("test1").settings(indexSettings).build(); - metaStateService.writeIndex("test_write", index); - assertThat(metaStateService.loadIndexState(index.getIndex()), equalTo(index)); - } + private static IndexMetaData indexMetaData(String name) { + return IndexMetaData.builder(name).settings( + Settings.builder() + .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .build() + ).build(); + } + + public void testWriteLoadIndex() throws Exception { + IndexMetaData index = indexMetaData("test1"); + metaStateService.writeIndex("test_write", index); + assertThat(metaStateService.loadIndexState(index.getIndex()), equalTo(index)); } public void testLoadMissingIndex() throws Exception { - try (NodeEnvironment env = newNodeEnvironment()) { - MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry()); - assertThat(metaStateService.loadIndexState(new Index("test1", "test1UUID")), nullValue()); - } + assertThat(metaStateService.loadIndexState(new Index("test1", "test1UUID")), nullValue()); } public void testWriteLoadGlobal() throws Exception { - try (NodeEnvironment env = newNodeEnvironment()) { - MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry()); - - MetaData metaData = MetaData.builder() - .persistentSettings(Settings.builder().put("test1", "value1").build()) - .build(); - metaStateService.writeGlobalState("test_write", metaData); - assertThat(metaStateService.loadGlobalState().persistentSettings(), equalTo(metaData.persistentSettings())); - } + MetaData metaData = MetaData.builder() + .persistentSettings(Settings.builder().put("test1", "value1").build()) + .build(); + metaStateService.writeGlobalState("test_write", metaData); + assertThat(metaStateService.loadGlobalState().persistentSettings(), equalTo(metaData.persistentSettings())); } public void testWriteGlobalStateWithIndexAndNoIndexIsLoaded() throws Exception { - try (NodeEnvironment env = newNodeEnvironment()) { - MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry()); - - MetaData metaData = MetaData.builder() - .persistentSettings(Settings.builder().put("test1", "value1").build()) - .build(); - IndexMetaData index = IndexMetaData.builder("test1").settings(indexSettings).build(); - MetaData metaDataWithIndex = MetaData.builder(metaData).put(index, true).build(); - - metaStateService.writeGlobalState("test_write", metaDataWithIndex); - assertThat(metaStateService.loadGlobalState().persistentSettings(), equalTo(metaData.persistentSettings())); - assertThat(metaStateService.loadGlobalState().hasIndex("test1"), equalTo(false)); - } + MetaData metaData = MetaData.builder() + .persistentSettings(Settings.builder().put("test1", "value1").build()) + .build(); + IndexMetaData index = indexMetaData("test1"); + MetaData metaDataWithIndex = MetaData.builder(metaData).put(index, true).build(); + + metaStateService.writeGlobalState("test_write", metaDataWithIndex); + assertThat(metaStateService.loadGlobalState().persistentSettings(), equalTo(metaData.persistentSettings())); + assertThat(metaStateService.loadGlobalState().hasIndex("test1"), equalTo(false)); } public void testLoadFullStateBWC() throws Exception { - try (NodeEnvironment env = newNodeEnvironment()) { - MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry()); - - IndexMetaData indexMetaData = IndexMetaData.builder("test1").settings(indexSettings).build(); - MetaData metaData = MetaData.builder() - .persistentSettings(Settings.builder().put("test1", "value1").build()) - .put(indexMetaData, true) - .build(); - - long globalGeneration = metaStateService.writeGlobalState("test_write", metaData); - long indexGeneration = metaStateService.writeIndex("test_write", indexMetaData); - - Tuple stateAndData = metaStateService.loadFullState(); - Manifest manifest = stateAndData.v1(); - assertThat(manifest.getGlobalGeneration(), equalTo(globalGeneration)); - assertThat(manifest.getIndexGenerations(), hasKey(indexMetaData.getIndex())); - assertThat(manifest.getIndexGenerations().get(indexMetaData.getIndex()), equalTo(indexGeneration)); - - MetaData loadedMetaData = stateAndData.v2(); - assertThat(loadedMetaData.persistentSettings(), equalTo(metaData.persistentSettings())); - assertThat(loadedMetaData.hasIndex("test1"), equalTo(true)); - assertThat(loadedMetaData.index("test1"), equalTo(indexMetaData)); - } + IndexMetaData indexMetaData = indexMetaData("test1"); + MetaData metaData = MetaData.builder() + .persistentSettings(Settings.builder().put("test1", "value1").build()) + .put(indexMetaData, true) + .build(); + + long globalGeneration = metaStateService.writeGlobalState("test_write", metaData); + long indexGeneration = metaStateService.writeIndex("test_write", indexMetaData); + + Tuple stateAndData = metaStateService.loadFullState(); + Manifest manifest = stateAndData.v1(); + assertThat(manifest.getGlobalGeneration(), equalTo(globalGeneration)); + assertThat(manifest.getIndexGenerations(), hasKey(indexMetaData.getIndex())); + assertThat(manifest.getIndexGenerations().get(indexMetaData.getIndex()), equalTo(indexGeneration)); + + MetaData loadedMetaData = stateAndData.v2(); + assertThat(loadedMetaData.persistentSettings(), equalTo(metaData.persistentSettings())); + assertThat(loadedMetaData.hasIndex("test1"), equalTo(true)); + assertThat(loadedMetaData.index("test1"), equalTo(indexMetaData)); } public void testLoadEmptyState() throws IOException { - try (NodeEnvironment env = newNodeEnvironment()) { - MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry()); - Tuple stateAndData = metaStateService.loadFullState(); - - Manifest manifest = stateAndData.v1(); - assertThat(manifest.getGlobalGeneration(), equalTo(-1L)); - assertThat(manifest.getIndexGenerations().entrySet(), empty()); - - MetaData metaData = stateAndData.v2(); - MetaData emptyMetaData = MetaData.builder().build(); - assertTrue(MetaData.isGlobalStateEquals(metaData, emptyMetaData)); - } + Tuple stateAndData = metaStateService.loadFullState(); + + Manifest manifest = stateAndData.v1(); + assertThat(manifest.getGlobalGeneration(), equalTo(-1L)); + assertThat(manifest.getIndexGenerations().entrySet(), empty()); + + MetaData metaData = stateAndData.v2(); + MetaData emptyMetaData = MetaData.builder().build(); + assertTrue(MetaData.isGlobalStateEquals(metaData, emptyMetaData)); } public void testLoadFullStateAndUpdate() throws IOException { - try (NodeEnvironment env = newNodeEnvironment()) { - MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry()); - - IndexMetaData index = IndexMetaData.builder("test1").settings(indexSettings).build(); - MetaData metaData = MetaData.builder() - .persistentSettings(Settings.builder().put("test1", "value1").build()) - .put(index, true) - .build(); - - long globalGeneration = metaStateService.writeGlobalState("first global state write", metaData); - long indexGeneration = metaStateService.writeIndex("first index state write", index); - - Manifest manifest = new Manifest(globalGeneration, new HashMap() {{ - put(index.getIndex(), indexGeneration); - }}); - - metaStateService.writeManifest("first meta state write", manifest); - - MetaData newMetaData = MetaData.builder() - .persistentSettings(Settings.builder().put("test1", "value2").build()) - .put(index, true) - .build(); - globalGeneration = metaStateService.writeGlobalState("second global state write", newMetaData); - - Tuple stateAndData = metaStateService.loadFullState(); - assertThat(stateAndData.v1(), equalTo(manifest)); - - MetaData loadedMetaData = stateAndData.v2(); - assertThat(loadedMetaData.persistentSettings(), equalTo(metaData.persistentSettings())); - assertThat(loadedMetaData.hasIndex("test1"), equalTo(true)); - assertThat(loadedMetaData.index("test1"), equalTo(index)); - - manifest = new Manifest(globalGeneration, new HashMap() {{ - put(index.getIndex(), indexGeneration); - }}); - - long metaStateGeneration = metaStateService.writeManifest("test", manifest); - metaStateService.cleanupGlobalState(globalGeneration); - metaStateService.cleanupIndex(index.getIndex(), indexGeneration); - metaStateService.cleanupMetaState(metaStateGeneration); - - stateAndData = metaStateService.loadFullState(); - assertThat(stateAndData.v1(), equalTo(manifest)); - - loadedMetaData = stateAndData.v2(); - assertThat(loadedMetaData.persistentSettings(), equalTo(newMetaData.persistentSettings())); - assertThat(loadedMetaData.hasIndex("test1"), equalTo(true)); - assertThat(loadedMetaData.index("test1"), equalTo(index)); - } + IndexMetaData index = indexMetaData("test1"); + MetaData metaData = MetaData.builder() + .persistentSettings(Settings.builder().put("test1", "value1").build()) + .put(index, true) + .build(); + + long globalGeneration = metaStateService.writeGlobalState("first global state write", metaData); + long indexGeneration = metaStateService.writeIndex("first index state write", index); + + Manifest manifest = new Manifest(globalGeneration, new HashMap() {{ + put(index.getIndex(), indexGeneration); + }}); + + metaStateService.writeManifest("first meta state write", manifest); + + MetaData newMetaData = MetaData.builder() + .persistentSettings(Settings.builder().put("test1", "value2").build()) + .put(index, true) + .build(); + globalGeneration = metaStateService.writeGlobalState("second global state write", newMetaData); + + Tuple stateAndData = metaStateService.loadFullState(); + assertThat(stateAndData.v1(), equalTo(manifest)); + + MetaData loadedMetaData = stateAndData.v2(); + assertThat(loadedMetaData.persistentSettings(), equalTo(metaData.persistentSettings())); + assertThat(loadedMetaData.hasIndex("test1"), equalTo(true)); + assertThat(loadedMetaData.index("test1"), equalTo(index)); + + manifest = new Manifest(globalGeneration, new HashMap() {{ + put(index.getIndex(), indexGeneration); + }}); + + long metaStateGeneration = metaStateService.writeManifest("test", manifest); + metaStateService.cleanupGlobalState(globalGeneration); + metaStateService.cleanupIndex(index.getIndex(), indexGeneration); + metaStateService.cleanupMetaState(metaStateGeneration); + + stateAndData = metaStateService.loadFullState(); + assertThat(stateAndData.v1(), equalTo(manifest)); + + loadedMetaData = stateAndData.v2(); + assertThat(loadedMetaData.persistentSettings(), equalTo(newMetaData.persistentSettings())); + assertThat(loadedMetaData.hasIndex("test1"), equalTo(true)); + assertThat(loadedMetaData.index("test1"), equalTo(index)); } } From 1c1136014aa198321b66215d02f87271c4de943a Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Wed, 7 Nov 2018 22:51:16 +0300 Subject: [PATCH 15/57] GatewayMetaState and MetaStateService bug fix --- .../gateway/GatewayMetaState.java | 58 ++++++++++--------- .../gateway/MetaStateService.java | 38 ++++++------ 2 files changed, 50 insertions(+), 46 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index f1dcc800e8279..f143f19fe816f 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -75,7 +75,8 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateA @Nullable //there is happens-before order between subsequent applyClusterState calls, hence no volatile modifier - private Tuple ourState; + private Manifest previousManifest; + private MetaData previousMetaData; public GatewayMetaState(Settings settings, NodeEnvironment nodeEnv, MetaStateService metaStateService, MetaDataIndexUpgradeService metaDataIndexUpgradeService, MetaDataUpgrader metaDataUpgrader) throws IOException { @@ -85,7 +86,7 @@ public GatewayMetaState(Settings settings, NodeEnvironment nodeEnv, MetaStateSer ensureNoPre019State(); ensureAtomicMoveSupported(); - maybeUpgradeMetaData(metaDataIndexUpgradeService, metaDataUpgrader); + upgradeMetaData(metaDataIndexUpgradeService, metaDataUpgrader); profileLoadMetaData(); } @@ -97,7 +98,7 @@ private void profileLoadMetaData() throws IOException { } } - private void maybeUpgradeMetaData(MetaDataIndexUpgradeService metaDataIndexUpgradeService, MetaDataUpgrader metaDataUpgrader) + private void upgradeMetaData(MetaDataIndexUpgradeService metaDataIndexUpgradeService, MetaDataUpgrader metaDataUpgrader) throws IOException { if (DiscoveryNode.isMasterNode(settings) || DiscoveryNode.isDataNode(settings)) { try { @@ -111,9 +112,10 @@ private void maybeUpgradeMetaData(MetaDataIndexUpgradeService metaDataIndexUpgra long globalStateGeneration = manifest.getGlobalGeneration(); if (globalStateGeneration != -1) { - List cleanupActions = new ArrayList<>(); // If globalStateGeneration is non-negative, it means we should have some metadata on disk // Always re-write it even if upgrade plugins do not upgrade it, to be sure it's properly persisted on all data path + + List cleanupActions = new ArrayList<>(); final MetaData upgradedMetaData = upgradeMetaData(metaData, metaDataIndexUpgradeService, metaDataUpgrader); if (MetaData.isGlobalStateEquals(metaData, upgradedMetaData) == false) { @@ -168,15 +170,19 @@ public void applyClusterState(ClusterChangedEvent event) { if (event.state().blocks().disableStatePersistence()) { // reset the current state, we need to start fresh... - ourState = null; + previousMetaData = null; + previousManifest = null; return; } try { - if (ourState == null) { - ourState = metaStateService.loadFullState(); + // if previousManifest is null we load it from disk + // we don't do the same for previous metadata, because our comparison can show that nothing was changed, + // but in fact it has changed + if (previousManifest == null) { + previousManifest = metaStateService.loadFullState().v1(); } - ourState = updateMetaData(event); + updateMetaData(event); } catch (WriteStateException e) { if (e.isDirty()) { logger.error("Fatal exception occurred when storing new meta data. Storage is dirty", e); @@ -194,7 +200,7 @@ public void applyClusterState(ClusterChangedEvent event) { * @throws IOException if IOException occurs. It's recommended for the callers of this method to handle {@link WriteStateException}, * which is subclass of {@link IOException} explicitly. See also {@link WriteStateException#isDirty()}. */ - private Tuple updateMetaData(ClusterChangedEvent event) throws IOException { + private void updateMetaData(ClusterChangedEvent event) throws IOException { ClusterState newState = event.state(); ClusterState previousState = event.previousState(); MetaData newMetaData = newState.metaData(); @@ -205,7 +211,9 @@ private Tuple updateMetaData(ClusterChangedEvent event) thro Manifest manifest = new Manifest(globalStateGeneration, newIndices); writeManifest(manifest, cleanupActions); performCleanup(cleanupActions); - return new Tuple<>(manifest, newMetaData); + + previousMetaData = newMetaData; + previousManifest = manifest; } private void performCleanup(List cleanupActions) { @@ -215,7 +223,7 @@ private void performCleanup(List cleanupActions) { } private void writeManifest(Manifest manifest, List cleanupActions) throws IOException { - if (manifest.equals(ourState.v1()) == false) { + if (manifest.equals(previousManifest) == false) { long generation = metaStateService.writeManifest("changed", manifest); cleanupActions.add(() -> metaStateService.cleanupMetaState(generation)); } @@ -223,14 +231,12 @@ private void writeManifest(Manifest manifest, List cleanupActions) thr private Map writeIndicesMetadata(ClusterState newState, ClusterState previousState, List cleanupActions) throws IOException { - MetaData previousMetadata = ourState.v2(); - Manifest previousMetastate = ourState.v1(); - Map previouslyWrittenIndices = previousMetastate.getIndexGenerations(); + Map previouslyWrittenIndices = previousManifest.getIndexGenerations(); Set relevantIndices = getRelevantIndices(newState, previousState, previouslyWrittenIndices.keySet()); Map newIndices = new HashMap<>(); - Iterable actions = resolveIndexMetaDataActions(previouslyWrittenIndices, relevantIndices, previousMetadata, + Iterable actions = resolveIndexMetaDataActions(previouslyWrittenIndices, relevantIndices, previousMetaData, newState.metaData()); for (IndexMetaDataAction action : actions) { @@ -242,12 +248,12 @@ private Map writeIndicesMetadata(ClusterState newState, ClusterStat } private long writeGlobalState(MetaData newMetaData, List cleanupActions) throws IOException { - if (ourState.v1().getGlobalGeneration() == -1 || MetaData.isGlobalStateEquals(ourState.v2(), newMetaData) == false) { + if (previousMetaData == null || MetaData.isGlobalStateEquals(previousMetaData, newMetaData) == false) { long generation = metaStateService.writeGlobalState("changed", newMetaData); cleanupActions.add(() -> metaStateService.cleanupGlobalState(generation)); return generation; } - return ourState.v1().getGlobalGeneration(); + return previousManifest.getGlobalGeneration(); } public static Set getRelevantIndices(ClusterState state, ClusterState previousState, Set previouslyWrittenIndices) { @@ -402,17 +408,15 @@ public static List resolveIndexMetaDataActions(Map actions = new ArrayList<>(); for (Index index : relevantIndices) { - Long generation = previouslyWrittenIndices.get(index); - IndexMetaData newIndexMetadata = newMetaData.getIndexSafe(index); - if (generation == null) { - actions.add(new WriteNewIndexMetaData(newIndexMetadata)); + IndexMetaData newIndexMetaData = newMetaData.getIndexSafe(index); + IndexMetaData previousIndexMetaData = previousMetaData == null ? null : previousMetaData.index(index); + + if (previouslyWrittenIndices.containsKey(index) == false || previousIndexMetaData == null) { + actions.add(new WriteNewIndexMetaData(newIndexMetaData)); + } else if (previousIndexMetaData.getVersion() != newIndexMetaData.getVersion()) { + actions.add(new WriteChangedIndexMetaData(previousIndexMetaData, newIndexMetaData)); } else { - IndexMetaData previousIndexMetadata = previousMetaData.index(index); - if (previousIndexMetadata.getVersion() != newIndexMetadata.getVersion()) { - actions.add(new WriteChangedIndexMetaData(previousIndexMetadata, newIndexMetadata)); - } else { - actions.add(new KeepPreviousGeneration(index, generation)); - } + actions.add(new KeepPreviousGeneration(index, previouslyWrittenIndices.get(index))); } } return actions; diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java index 19b6cce1e7aa6..2a7293a62dfb2 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java @@ -96,33 +96,33 @@ Tuple loadFullState() throws IOException { */ private Tuple loadFullStateBWC() throws IOException { Map indices = new HashMap<>(); + MetaData.Builder metaDataBuilder = MetaData.builder(); + Tuple metaDataAndGeneration = MetaData.FORMAT.loadLatestStateWithGeneration(logger, namedXContentRegistry, nodeEnv.nodeDataPaths()); MetaData globalMetaData = metaDataAndGeneration.v1(); long globalStateGeneration = metaDataAndGeneration.v2(); boolean isFreshStartup = globalMetaData == null; - if (isFreshStartup) { + if (isFreshStartup == false) { assert Version.CURRENT.major < 8 : "failed to find manifest file, which is mandatory staring with Elasticsearch version 8.0"; - } - MetaData.Builder metaDataBuilder; - if (globalMetaData != null) { - metaDataBuilder = MetaData.builder(globalMetaData); - } else { - metaDataBuilder = MetaData.builder(); - } - for (String indexFolderName : nodeEnv.availableIndexFolders()) { - Tuple indexMetaDataAndGeneration = - IndexMetaData.FORMAT.loadLatestStateWithGeneration(logger, namedXContentRegistry, - nodeEnv.resolveIndexFolder(indexFolderName)); - IndexMetaData indexMetaData = indexMetaDataAndGeneration.v1(); - long generation = indexMetaDataAndGeneration.v2(); - if (indexMetaData != null) { - indices.put(indexMetaData.getIndex(), generation); - metaDataBuilder.put(indexMetaData, false); - } else { - logger.debug("[{}] failed to find metadata for existing index location", indexFolderName); + if (globalMetaData != null) { + metaDataBuilder = MetaData.builder(globalMetaData); + } + + for (String indexFolderName : nodeEnv.availableIndexFolders()) { + Tuple indexMetaDataAndGeneration = + IndexMetaData.FORMAT.loadLatestStateWithGeneration(logger, namedXContentRegistry, + nodeEnv.resolveIndexFolder(indexFolderName)); + IndexMetaData indexMetaData = indexMetaDataAndGeneration.v1(); + long generation = indexMetaDataAndGeneration.v2(); + if (indexMetaData != null) { + indices.put(indexMetaData.getIndex(), generation); + metaDataBuilder.put(indexMetaData, false); + } else { + logger.debug("[{}] failed to find metadata for existing index location", indexFolderName); + } } } Manifest manifest = new Manifest(globalStateGeneration, indices); From a2c7226cc4c3df87174c420746a68c73968c20a0 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Fri, 9 Nov 2018 14:06:52 +0300 Subject: [PATCH 16/57] Manipulation -> parsing/generation --- .../main/java/org/elasticsearch/cluster/metadata/Manifest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java index 63b1e2b9be796..81f9234985751 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java @@ -98,7 +98,7 @@ public Manifest fromXContent(XContentParser parser) throws IOException { /* - * Code below this comment is for XContent manipulation + * Code below this comment is for XContent parsing/generation */ private static final ParseField GENERATION_PARSE_FIELD = new ParseField("generation"); From 96029ba41f3802fc455d790a14504b8961840196 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Fri, 9 Nov 2018 14:11:25 +0300 Subject: [PATCH 17/57] Add TODO to GatewayMetaState --- .../main/java/org/elasticsearch/gateway/GatewayMetaState.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index f143f19fe816f..6ce7a0b1d7be8 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -84,8 +84,8 @@ public GatewayMetaState(Settings settings, NodeEnvironment nodeEnv, MetaStateSer this.nodeEnv = nodeEnv; this.metaStateService = metaStateService; - ensureNoPre019State(); - ensureAtomicMoveSupported(); + ensureNoPre019State(); //TODO remove this check, it's Elasticsearch version 7 already + ensureAtomicMoveSupported(); //TODO move this check to NodeEnvironment, because it's related to all types of metadata upgradeMetaData(metaDataIndexUpgradeService, metaDataUpgrader); profileLoadMetaData(); } From db6da1151570f9ad452831cc8affde37cd69a205 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Fri, 9 Nov 2018 14:21:20 +0300 Subject: [PATCH 18/57] Do not write non-upgraded metadata on the startup --- .../gateway/GatewayMetaState.java | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index 6ce7a0b1d7be8..19bdd98bfa559 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -113,30 +113,28 @@ private void upgradeMetaData(MetaDataIndexUpgradeService metaDataIndexUpgradeSer if (globalStateGeneration != -1) { // If globalStateGeneration is non-negative, it means we should have some metadata on disk - // Always re-write it even if upgrade plugins do not upgrade it, to be sure it's properly persisted on all data path + // We don't re-write metadata if it's not upgraded by upgrade plugins, because + // if there is manifest file, it means metadata is properly persisted to all data paths + // if there is no manifest file (upgrade from 6.x to 7.x) metadata might be missing on some data paths, + // but anyway we will re-write it as soon as we receive first ClusterState List cleanupActions = new ArrayList<>(); final MetaData upgradedMetaData = upgradeMetaData(metaData, metaDataIndexUpgradeService, metaDataUpgrader); if (MetaData.isGlobalStateEquals(metaData, upgradedMetaData) == false) { globalStateGeneration = metaStateService.writeGlobalState("upgrade", upgradedMetaData); - } else { - globalStateGeneration = metaStateService.writeGlobalState("startup", metaData); + final long currentGlobalStateGeneration = globalStateGeneration; + cleanupActions.add(() -> metaStateService.cleanupGlobalState(currentGlobalStateGeneration)); } - final long currentGlobalStateGeneration = globalStateGeneration; - cleanupActions.add(() -> metaStateService.cleanupGlobalState(currentGlobalStateGeneration)); - Map indices = new HashMap<>(); + Map indices = new HashMap<>(manifest.getIndexGenerations()); + for (IndexMetaData indexMetaData : upgradedMetaData) { - long generation; if (metaData.hasIndexMetaData(indexMetaData) == false) { - generation = metaStateService.writeIndex("upgrade", indexMetaData); - } else { - generation = metaStateService.writeIndex("startup", indexMetaData); + final long generation = metaStateService.writeIndex("upgrade", indexMetaData); + cleanupActions.add(() -> metaStateService.cleanupIndex(indexMetaData.getIndex(), generation)); + indices.put(indexMetaData.getIndex(), generation); } - final long currentGeneration = generation; - cleanupActions.add(() -> metaStateService.cleanupIndex(indexMetaData.getIndex(), currentGeneration)); - indices.put(indexMetaData.getIndex(), generation); } final long metaStateGeneration = From 541c393498fe6808950072b40de2a01803767d1c Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Fri, 9 Nov 2018 14:29:08 +0300 Subject: [PATCH 19/57] isDataOrMasterNode --- .../elasticsearch/gateway/GatewayMetaState.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index 19bdd98bfa559..0c9508d0dfbfd 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -91,7 +91,7 @@ public GatewayMetaState(Settings settings, NodeEnvironment nodeEnv, MetaStateSer } private void profileLoadMetaData() throws IOException { - if (DiscoveryNode.isMasterNode(settings) || DiscoveryNode.isDataNode(settings)) { + if (isMasterOrDataNode()) { long startNS = System.nanoTime(); metaStateService.loadFullState(); logger.debug("took {} to load state", TimeValue.timeValueMillis(TimeValue.nsecToMSec(System.nanoTime() - startNS))); @@ -100,7 +100,7 @@ private void profileLoadMetaData() throws IOException { private void upgradeMetaData(MetaDataIndexUpgradeService metaDataIndexUpgradeService, MetaDataUpgrader metaDataUpgrader) throws IOException { - if (DiscoveryNode.isMasterNode(settings) || DiscoveryNode.isDataNode(settings)) { + if (isMasterOrDataNode()) { try { final Tuple metaStateAndData = metaStateService.loadFullState(); final Manifest manifest = metaStateAndData.v1(); @@ -149,8 +149,12 @@ private void upgradeMetaData(MetaDataIndexUpgradeService metaDataIndexUpgradeSer } } + private boolean isMasterOrDataNode() { + return DiscoveryNode.isMasterNode(settings) || DiscoveryNode.isDataNode(settings); + } + private void ensureAtomicMoveSupported() throws IOException { - if (DiscoveryNode.isMasterNode(settings) || DiscoveryNode.isDataNode(settings)) { + if (isMasterOrDataNode()) { nodeEnv.ensureAtomicMoveSupported(); } } @@ -161,8 +165,7 @@ public MetaData loadMetaData() throws IOException { @Override public void applyClusterState(ClusterChangedEvent event) { - ClusterState newState = event.state(); - if (newState.nodes().getLocalNode().isMasterNode() == false && newState.nodes().getLocalNode().isDataNode() == false) { + if (isMasterOrDataNode() == false) { return; } @@ -276,7 +279,7 @@ private void ensureNoPre019State() throws IOException { if (DiscoveryNode.isDataNode(settings)) { ensureNoPre019ShardState(); } - if (DiscoveryNode.isMasterNode(settings) || DiscoveryNode.isDataNode(settings)) { + if (isMasterOrDataNode()) { ensureNoPre019MetadataFiles(); } } From b5775cf5f4a41b9e7cbafd997385706216eb41fe Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Fri, 9 Nov 2018 14:40:30 +0300 Subject: [PATCH 20/57] Load manifest only on data node, create a method for it --- .../elasticsearch/gateway/GatewayMetaState.java | 9 +++------ .../elasticsearch/gateway/MetaStateService.java | 17 ++++++++++------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index 0c9508d0dfbfd..237d9c02765ee 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -177,12 +177,6 @@ public void applyClusterState(ClusterChangedEvent event) { } try { - // if previousManifest is null we load it from disk - // we don't do the same for previous metadata, because our comparison can show that nothing was changed, - // but in fact it has changed - if (previousManifest == null) { - previousManifest = metaStateService.loadFullState().v1(); - } updateMetaData(event); } catch (WriteStateException e) { if (e.isDirty()) { @@ -232,6 +226,9 @@ private void writeManifest(Manifest manifest, List cleanupActions) thr private Map writeIndicesMetadata(ClusterState newState, ClusterState previousState, List cleanupActions) throws IOException { + if (previousManifest == null) { + previousManifest = metaStateService.loadManifest(); + } Map previouslyWrittenIndices = previousManifest.getIndexGenerations(); Set relevantIndices = getRelevantIndices(newState, previousState, previouslyWrittenIndices.keySet()); diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java index 2a7293a62dfb2..65d3302ad1038 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java @@ -63,7 +63,7 @@ public MetaStateService(Settings settings, NodeEnvironment nodeEnv, NamedXConten * @throws IOException if some IOException when loading files occurs or there is no metadata referenced by manifest file. */ Tuple loadFullState() throws IOException { - final Manifest manifest = loadMetaState(); + final Manifest manifest = loadManifest(); if (manifest == null) { return loadFullStateBWC(); } @@ -137,10 +137,6 @@ public IndexMetaData loadIndexState(Index index) throws IOException { return IndexMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.indexPaths(index)); } - private Manifest loadMetaState() throws IOException { - return Manifest.FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.nodeDataPaths()); - } - /** * Loads all indices states available on disk */ @@ -165,6 +161,13 @@ List loadIndicesStates(Predicate excludeIndexPathIdsPredi return indexMetaDataList; } + /** + * Loads Manifest file from disk, returns null if there is no manifest file. + */ + public Manifest loadManifest() throws IOException { + return Manifest.FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.nodeDataPaths()); + } + /** * Loads the global state, *without* index state, see {@link #loadFullState()} for that. */ @@ -259,7 +262,7 @@ public void cleanupMetaState(long currentGeneration) { */ public void writeIndexAndUpdateManifest(String reason, IndexMetaData metaData) throws IOException { long generation = writeIndex(reason, metaData); - Manifest manifest = loadMetaState(); + Manifest manifest = loadManifest(); Map indices = new HashMap<>(manifest.getIndexGenerations()); indices.put(metaData.getIndex(), generation); manifest = new Manifest(manifest.getGlobalGeneration(), indices); @@ -273,7 +276,7 @@ public void writeIndexAndUpdateManifest(String reason, IndexMetaData metaData) t */ public void writeGlobalStateAndUpdateManifest(String reason, MetaData metaData) throws IOException { long generation = writeGlobalState(reason, metaData); - Manifest manifest = loadMetaState(); + Manifest manifest = loadManifest(); manifest = new Manifest(generation, manifest.getIndexGenerations()); long metaStateGeneration = writeManifest(reason, manifest); cleanupGlobalState(generation); From 625d2bf3b0d0ae5ccb91b44bd68358c8e3b1d741 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Fri, 9 Nov 2018 14:41:06 +0300 Subject: [PATCH 21/57] Remove "Storage is not dirty" --- .../main/java/org/elasticsearch/gateway/GatewayMetaState.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index 237d9c02765ee..3a27221944bdf 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -182,7 +182,7 @@ public void applyClusterState(ClusterChangedEvent event) { if (e.isDirty()) { logger.error("Fatal exception occurred when storing new meta data. Storage is dirty", e); } else { - logger.warn("Exception occurred when storing new meta data. Storage is not dirty", e); + logger.warn("Exception occurred when storing new meta data.", e); } } catch (Exception e) { logger.warn("Exception occurred when storing new meta data", e); From b6872a8e8097c79ce7268c70ff90b97af0515088 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Fri, 9 Nov 2018 14:42:27 +0300 Subject: [PATCH 22/57] Inline perform cleanup --- .../elasticsearch/gateway/GatewayMetaState.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index 3a27221944bdf..3ede425fdbe2e 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -140,7 +140,10 @@ private void upgradeMetaData(MetaDataIndexUpgradeService metaDataIndexUpgradeSer final long metaStateGeneration = metaStateService.writeManifest("startup", new Manifest(globalStateGeneration, indices)); cleanupActions.add(() -> metaStateService.cleanupMetaState(metaStateGeneration)); - performCleanup(cleanupActions); + + for (Runnable action : cleanupActions) { + action.run(); + } } } catch (Exception e) { logger.error("failed to read or re-write local state, exiting...", e); @@ -205,16 +208,13 @@ private void updateMetaData(ClusterChangedEvent event) throws IOException { Map newIndices = writeIndicesMetadata(newState, previousState, cleanupActions); Manifest manifest = new Manifest(globalStateGeneration, newIndices); writeManifest(manifest, cleanupActions); - performCleanup(cleanupActions); - previousMetaData = newMetaData; - previousManifest = manifest; - } - - private void performCleanup(List cleanupActions) { for (Runnable action : cleanupActions) { action.run(); } + + previousMetaData = newMetaData; + previousManifest = manifest; } private void writeManifest(Manifest manifest, List cleanupActions) throws IOException { From 2f1f7f744f2389b082eb6d7a322209139829e23c Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Fri, 9 Nov 2018 14:47:27 +0300 Subject: [PATCH 23/57] Add index name to IOException --- .../main/java/org/elasticsearch/gateway/MetaStateService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java index 65d3302ad1038..f86f8ff9d1841 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java @@ -84,7 +84,7 @@ Tuple loadFullState() throws IOException { if (indexMetaData != null) { metaDataBuilder.put(indexMetaData, false); } else { - throw new IOException("failed to find metadata for existing index [location: " + indexFolderName + + throw new IOException("failed to find metadata for existing index " + index.getName() + " [location: " + indexFolderName + ", generation: " + generation + "]"); } } From 75045f085de8575f0c51891c721c022d2a3d4754 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Fri, 9 Nov 2018 14:50:38 +0300 Subject: [PATCH 24/57] Used by tests javadoc --- .../main/java/org/elasticsearch/gateway/MetaStateService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java index f86f8ff9d1841..4155358aed441 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java @@ -259,6 +259,7 @@ public void cleanupMetaState(long currentGeneration) { /** * Writes index metadata and updates manifest file accordingly. + * Used by tests. */ public void writeIndexAndUpdateManifest(String reason, IndexMetaData metaData) throws IOException { long generation = writeIndex(reason, metaData); @@ -273,6 +274,7 @@ public void writeIndexAndUpdateManifest(String reason, IndexMetaData metaData) t /** * Writes global metadata and updates manifest file accordingly. + * Used by tests. */ public void writeGlobalStateAndUpdateManifest(String reason, MetaData metaData) throws IOException { long generation = writeGlobalState(reason, metaData); From 466ad6f70ea5c164a7a5286709f3060e60a0e475 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Fri, 9 Nov 2018 15:07:21 +0300 Subject: [PATCH 25/57] Rename tests --- .../elasticsearch/gateway/GatewayMetaStateTests.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java index 13d084d990c1b..2077025ac5148 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java @@ -149,7 +149,7 @@ private IndexMetaData createIndexMetaData(String name) { build(); } - public void testGetRelevantIndices_master_unassignedShards() { + public void testGetRelevantIndicesWithUnassignedShardsOnMasterEligibleNode() { IndexMetaData indexMetaData = createIndexMetaData("test"); Set indices = GatewayMetaState.getRelevantIndices( clusterStateWithUnassignedIndex(indexMetaData, true), @@ -158,7 +158,7 @@ public void testGetRelevantIndices_master_unassignedShards() { assertThat(indices.size(), equalTo(1)); } - public void testGetRelevantIndices_data_unassignedShards() { + public void testGetRelevantIndicesWithUnassignedShardsOnDataOnlyNode() { IndexMetaData indexMetaData = createIndexMetaData("test"); Set indices = GatewayMetaState.getRelevantIndices( clusterStateWithUnassignedIndex(indexMetaData, false), @@ -167,7 +167,7 @@ public void testGetRelevantIndices_data_unassignedShards() { assertThat(indices.size(), equalTo(0)); } - public void testGetRelevantIndices_assignedShards() { + public void testGetRelevantIndicesWithAssignedShards() { IndexMetaData indexMetaData = createIndexMetaData("test"); boolean masterEligible = randomBoolean(); Set indices = GatewayMetaState.getRelevantIndices( @@ -177,7 +177,7 @@ public void testGetRelevantIndices_assignedShards() { assertThat(indices.size(), equalTo(1)); } - public void testGetRelevantIndices_data_isClosedPrevWritten() { + public void testGetRelevantIndicesForClosedPrevWrittenIndexOnDataOnlyNode() { IndexMetaData indexMetaData = createIndexMetaData("test"); Set indices = GatewayMetaState.getRelevantIndices( clusterStateWithClosedIndex(indexMetaData, false), @@ -186,7 +186,7 @@ public void testGetRelevantIndices_data_isClosedPrevWritten() { assertThat(indices.size(), equalTo(1)); } - public void testGetRelevantIndices_data_isClosedPrevNotWritten() { + public void testGetRelevantIndicesForClosedPrevNotWrittenIndexOnDataOnlyNode() { IndexMetaData indexMetaData = createIndexMetaData("test"); Set indices = GatewayMetaState.getRelevantIndices( clusterStateWithJustOpenedIndex(indexMetaData, false), @@ -195,7 +195,7 @@ public void testGetRelevantIndices_data_isClosedPrevNotWritten() { assertThat(indices.size(), equalTo(0)); } - public void testGetRelevantIndices_data_wasClosedPrevWritten() { + public void testGetRelevantIndicesForWasClosedPrevWrittenIndexOnDataOnlyNode() { IndexMetaData indexMetaData = createIndexMetaData("test"); Set indices = GatewayMetaState.getRelevantIndices( clusterStateWithJustOpenedIndex(indexMetaData, false), From 4aacfce40688b04b1ec32ced83ca2da2e265f06d Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Fri, 9 Nov 2018 15:29:13 +0300 Subject: [PATCH 26/57] Ignore missing global metadata if there is no manifest file --- .../cluster/metadata/Manifest.java | 4 ++ .../gateway/GatewayMetaState.java | 52 +++++++++---------- .../gateway/MetaStateService.java | 38 +++++++------- 3 files changed, 48 insertions(+), 46 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java index 81f9234985751..d5b6b3f3e8db7 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java @@ -140,6 +140,10 @@ public static Manifest fromXContent(XContentParser parser) throws IOException { return PARSER.parse(parser, null); } + public boolean isEmpty() { + return globalGeneration == -1 || indexGenerations.isEmpty(); + } + private static final class IndexEntry implements ToXContentFragment { private static final ParseField INDEX_GENERATION_PARSE_FIELD = new ParseField("generation"); private static final ParseField INDEX_PARSE_FIELD = new ParseField("index"); diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index 3ede425fdbe2e..4735645e9b453 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -109,41 +109,39 @@ private void upgradeMetaData(MetaDataIndexUpgradeService metaDataIndexUpgradeSer // We finished global state validation and successfully checked all indices for backward compatibility // and found no non-upgradable indices, which means the upgrade can continue. // Now it's safe to overwrite global and index metadata. - long globalStateGeneration = manifest.getGlobalGeneration(); - - if (globalStateGeneration != -1) { - // If globalStateGeneration is non-negative, it means we should have some metadata on disk - // We don't re-write metadata if it's not upgraded by upgrade plugins, because - // if there is manifest file, it means metadata is properly persisted to all data paths - // if there is no manifest file (upgrade from 6.x to 7.x) metadata might be missing on some data paths, - // but anyway we will re-write it as soon as we receive first ClusterState + // We don't re-write metadata if it's not upgraded by upgrade plugins, because + // if there is manifest file, it means metadata is properly persisted to all data paths + // if there is no manifest file (upgrade from 6.x to 7.x) metadata might be missing on some data paths, + // but anyway we will re-write it as soon as we receive first ClusterState + List cleanupActions = new ArrayList<>(); + final MetaData upgradedMetaData = upgradeMetaData(metaData, metaDataIndexUpgradeService, metaDataUpgrader); - List cleanupActions = new ArrayList<>(); - final MetaData upgradedMetaData = upgradeMetaData(metaData, metaDataIndexUpgradeService, metaDataUpgrader); + long globalStateGeneration = manifest.getGlobalGeneration(); + if (MetaData.isGlobalStateEquals(metaData, upgradedMetaData) == false) { + globalStateGeneration = metaStateService.writeGlobalState("upgrade", upgradedMetaData); + final long currentGlobalStateGeneration = globalStateGeneration; + cleanupActions.add(() -> metaStateService.cleanupGlobalState(currentGlobalStateGeneration)); + } - if (MetaData.isGlobalStateEquals(metaData, upgradedMetaData) == false) { - globalStateGeneration = metaStateService.writeGlobalState("upgrade", upgradedMetaData); - final long currentGlobalStateGeneration = globalStateGeneration; - cleanupActions.add(() -> metaStateService.cleanupGlobalState(currentGlobalStateGeneration)); + Map indices = new HashMap<>(manifest.getIndexGenerations()); + for (IndexMetaData indexMetaData : upgradedMetaData) { + if (metaData.hasIndexMetaData(indexMetaData) == false) { + final long generation = metaStateService.writeIndex("upgrade", indexMetaData); + cleanupActions.add(() -> metaStateService.cleanupIndex(indexMetaData.getIndex(), generation)); + indices.put(indexMetaData.getIndex(), generation); } + } - Map indices = new HashMap<>(manifest.getIndexGenerations()); - - for (IndexMetaData indexMetaData : upgradedMetaData) { - if (metaData.hasIndexMetaData(indexMetaData) == false) { - final long generation = metaStateService.writeIndex("upgrade", indexMetaData); - cleanupActions.add(() -> metaStateService.cleanupIndex(indexMetaData.getIndex(), generation)); - indices.put(indexMetaData.getIndex(), generation); - } - } + final Manifest newManifest = new Manifest(globalStateGeneration, indices); + if (newManifest.isEmpty() == false) { final long metaStateGeneration = - metaStateService.writeManifest("startup", new Manifest(globalStateGeneration, indices)); + metaStateService.writeManifest("startup", newManifest); cleanupActions.add(() -> metaStateService.cleanupMetaState(metaStateGeneration)); + } - for (Runnable action : cleanupActions) { - action.run(); - } + for (Runnable action : cleanupActions) { + action.run(); } } catch (Exception e) { logger.error("failed to read or re-write local state, exiting...", e); diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java index 4155358aed441..9d8a728b949a1 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java @@ -96,35 +96,35 @@ Tuple loadFullState() throws IOException { */ private Tuple loadFullStateBWC() throws IOException { Map indices = new HashMap<>(); - MetaData.Builder metaDataBuilder = MetaData.builder(); + MetaData.Builder metaDataBuilder; Tuple metaDataAndGeneration = MetaData.FORMAT.loadLatestStateWithGeneration(logger, namedXContentRegistry, nodeEnv.nodeDataPaths()); MetaData globalMetaData = metaDataAndGeneration.v1(); long globalStateGeneration = metaDataAndGeneration.v2(); - boolean isFreshStartup = globalMetaData == null; - if (isFreshStartup == false) { + if (globalMetaData != null) { + metaDataBuilder = MetaData.builder(globalMetaData); assert Version.CURRENT.major < 8 : "failed to find manifest file, which is mandatory staring with Elasticsearch version 8.0"; + } else { + metaDataBuilder = MetaData.builder(); + } - if (globalMetaData != null) { - metaDataBuilder = MetaData.builder(globalMetaData); - } - - for (String indexFolderName : nodeEnv.availableIndexFolders()) { - Tuple indexMetaDataAndGeneration = - IndexMetaData.FORMAT.loadLatestStateWithGeneration(logger, namedXContentRegistry, - nodeEnv.resolveIndexFolder(indexFolderName)); - IndexMetaData indexMetaData = indexMetaDataAndGeneration.v1(); - long generation = indexMetaDataAndGeneration.v2(); - if (indexMetaData != null) { - indices.put(indexMetaData.getIndex(), generation); - metaDataBuilder.put(indexMetaData, false); - } else { - logger.debug("[{}] failed to find metadata for existing index location", indexFolderName); - } + for (String indexFolderName : nodeEnv.availableIndexFolders()) { + Tuple indexMetaDataAndGeneration = + IndexMetaData.FORMAT.loadLatestStateWithGeneration(logger, namedXContentRegistry, + nodeEnv.resolveIndexFolder(indexFolderName)); + assert Version.CURRENT.major < 8 : "failed to find manifest file, which is mandatory staring with Elasticsearch version 8.0"; + IndexMetaData indexMetaData = indexMetaDataAndGeneration.v1(); + long generation = indexMetaDataAndGeneration.v2(); + if (indexMetaData != null) { + indices.put(indexMetaData.getIndex(), generation); + metaDataBuilder.put(indexMetaData, false); + } else { + logger.debug("[{}] failed to find metadata for existing index location", indexFolderName); } } + Manifest manifest = new Manifest(globalStateGeneration, indices); return new Tuple<>(manifest, metaDataBuilder.build()); } From 7ff091e472139237c76ec58127dfc383bdacc75f Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Fri, 9 Nov 2018 15:39:39 +0300 Subject: [PATCH 27/57] Empty manifest instead of null --- .../org/elasticsearch/cluster/metadata/Manifest.java | 9 ++++++++- .../org/elasticsearch/gateway/MetaStateService.java | 10 +++++++--- .../elasticsearch/cluster/metadata/ManifestTests.java | 5 +++++ 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java index d5b6b3f3e8db7..3f60a61b78802 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java @@ -31,6 +31,7 @@ import java.io.IOException; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -44,6 +45,8 @@ * Index metadata generation could be obtained by calling {@link #getIndexGenerations()}. */ public class Manifest implements ToXContentFragment { + private static final long MISSING_GLOBAL_GENERATION = -1; + private final long globalGeneration; private final Map indexGenerations; @@ -141,7 +144,11 @@ public static Manifest fromXContent(XContentParser parser) throws IOException { } public boolean isEmpty() { - return globalGeneration == -1 || indexGenerations.isEmpty(); + return globalGeneration == MISSING_GLOBAL_GENERATION || indexGenerations.isEmpty(); + } + + public static Manifest empty() { + return new Manifest(MISSING_GLOBAL_GENERATION, Collections.emptyMap()); } private static final class IndexEntry implements ToXContentFragment { diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java index 9d8a728b949a1..60b6e4666c16a 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java @@ -64,7 +64,7 @@ public MetaStateService(Settings settings, NodeEnvironment nodeEnv, NamedXConten */ Tuple loadFullState() throws IOException { final Manifest manifest = loadManifest(); - if (manifest == null) { + if (manifest.isEmpty()) { return loadFullStateBWC(); } final MetaData.Builder metaDataBuilder; @@ -162,10 +162,14 @@ List loadIndicesStates(Predicate excludeIndexPathIdsPredi } /** - * Loads Manifest file from disk, returns null if there is no manifest file. + * Loads Manifest file from disk, returns Manifest.empty() if there is no manifest file. */ public Manifest loadManifest() throws IOException { - return Manifest.FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.nodeDataPaths()); + Manifest manifest = Manifest.FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.nodeDataPaths()); + if (manifest == null) { + manifest = Manifest.empty(); + } + return manifest; } /** diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ManifestTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ManifestTests.java index 6c2bdbe8ea0cd..4d1b92b2b2dc9 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ManifestTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ManifestTests.java @@ -99,4 +99,9 @@ public void testXContent() throws IOException { assertThat(Manifest.fromXContent(parser), equalTo(state)); } } + + public void testEmptyManifest() { + assertTrue(Manifest.empty().isEmpty()); + assertFalse(randomManifest().isEmpty()); + } } \ No newline at end of file From f57067d0d6bb78e4b020f59293fd95258adf8af8 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Fri, 9 Nov 2018 16:14:26 +0300 Subject: [PATCH 28/57] Remove unused import --- .../main/java/org/elasticsearch/cluster/metadata/Manifest.java | 1 - .../src/test/java/org/elasticsearch/gateway/QuorumGatewayIT.java | 1 - 2 files changed, 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java index 3f60a61b78802..f8b7917d3ce1f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java @@ -31,7 +31,6 @@ import java.io.IOException; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; diff --git a/server/src/test/java/org/elasticsearch/gateway/QuorumGatewayIT.java b/server/src/test/java/org/elasticsearch/gateway/QuorumGatewayIT.java index cbd2b656875e4..c31b095b32323 100644 --- a/server/src/test/java/org/elasticsearch/gateway/QuorumGatewayIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/QuorumGatewayIT.java @@ -27,7 +27,6 @@ import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.InternalTestCluster; -import org.elasticsearch.test.InternalTestCluster.RestartCallback; import java.util.concurrent.TimeUnit; From 3de2060bee2d65fd0d25b1f8153a25cfa430f6f3 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Mon, 12 Nov 2018 12:32:46 +0300 Subject: [PATCH 29/57] isEmpty fix --- .../main/java/org/elasticsearch/cluster/metadata/Manifest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java index f8b7917d3ce1f..fa517ece2062e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java @@ -143,7 +143,7 @@ public static Manifest fromXContent(XContentParser parser) throws IOException { } public boolean isEmpty() { - return globalGeneration == MISSING_GLOBAL_GENERATION || indexGenerations.isEmpty(); + return globalGeneration == MISSING_GLOBAL_GENERATION && indexGenerations.isEmpty(); } public static Manifest empty() { From 14aff803d1d78214470dbaaa514812736cbe0751 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Mon, 12 Nov 2018 12:36:33 +0300 Subject: [PATCH 30/57] happens-before -> single thread comment --- .../main/java/org/elasticsearch/gateway/GatewayMetaState.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index 4735645e9b453..51447493cd6e1 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -74,7 +74,7 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateA private final MetaStateService metaStateService; @Nullable - //there is happens-before order between subsequent applyClusterState calls, hence no volatile modifier + //there is a single thread executing applyClusterState calls, hence no volatile modifier private Manifest previousManifest; private MetaData previousMetaData; From 6b1accb64f827dd9175345da29ef96ecc5fe7303 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Mon, 12 Nov 2018 12:38:15 +0300 Subject: [PATCH 31/57] Make globalStateGeneration final --- .../java/org/elasticsearch/gateway/GatewayMetaState.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index 51447493cd6e1..81bdadff3b832 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -116,11 +116,12 @@ private void upgradeMetaData(MetaDataIndexUpgradeService metaDataIndexUpgradeSer List cleanupActions = new ArrayList<>(); final MetaData upgradedMetaData = upgradeMetaData(metaData, metaDataIndexUpgradeService, metaDataUpgrader); - long globalStateGeneration = manifest.getGlobalGeneration(); + final long globalStateGeneration; if (MetaData.isGlobalStateEquals(metaData, upgradedMetaData) == false) { globalStateGeneration = metaStateService.writeGlobalState("upgrade", upgradedMetaData); - final long currentGlobalStateGeneration = globalStateGeneration; - cleanupActions.add(() -> metaStateService.cleanupGlobalState(currentGlobalStateGeneration)); + cleanupActions.add(() -> metaStateService.cleanupGlobalState(globalStateGeneration)); + } else { + globalStateGeneration = manifest.getGlobalGeneration(); } Map indices = new HashMap<>(manifest.getIndexGenerations()); From 0e0ec1c4cd5b64e0ed5d8333e3c701f4e4c8429d Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Mon, 12 Nov 2018 12:43:00 +0300 Subject: [PATCH 32/57] Remove WriteStateException handling --- .../java/org/elasticsearch/gateway/GatewayMetaState.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index 81bdadff3b832..71f7b8eafe091 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -180,12 +180,6 @@ public void applyClusterState(ClusterChangedEvent event) { try { updateMetaData(event); - } catch (WriteStateException e) { - if (e.isDirty()) { - logger.error("Fatal exception occurred when storing new meta data. Storage is dirty", e); - } else { - logger.warn("Exception occurred when storing new meta data.", e); - } } catch (Exception e) { logger.warn("Exception occurred when storing new meta data", e); } From f463893740638d96683e50cf7424e710a4755359 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Mon, 12 Nov 2018 12:44:48 +0300 Subject: [PATCH 33/57] loadManifest -> loadManifestOrEmpty --- .../java/org/elasticsearch/gateway/GatewayMetaState.java | 2 +- .../java/org/elasticsearch/gateway/MetaStateService.java | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index 71f7b8eafe091..5817974c35871 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -220,7 +220,7 @@ private void writeManifest(Manifest manifest, List cleanupActions) thr private Map writeIndicesMetadata(ClusterState newState, ClusterState previousState, List cleanupActions) throws IOException { if (previousManifest == null) { - previousManifest = metaStateService.loadManifest(); + previousManifest = metaStateService.loadManifestOrEmpty(); } Map previouslyWrittenIndices = previousManifest.getIndexGenerations(); Set relevantIndices = getRelevantIndices(newState, previousState, previouslyWrittenIndices.keySet()); diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java index 60b6e4666c16a..bd740e9e39ecf 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java @@ -63,7 +63,7 @@ public MetaStateService(Settings settings, NodeEnvironment nodeEnv, NamedXConten * @throws IOException if some IOException when loading files occurs or there is no metadata referenced by manifest file. */ Tuple loadFullState() throws IOException { - final Manifest manifest = loadManifest(); + final Manifest manifest = loadManifestOrEmpty(); if (manifest.isEmpty()) { return loadFullStateBWC(); } @@ -164,7 +164,7 @@ List loadIndicesStates(Predicate excludeIndexPathIdsPredi /** * Loads Manifest file from disk, returns Manifest.empty() if there is no manifest file. */ - public Manifest loadManifest() throws IOException { + public Manifest loadManifestOrEmpty() throws IOException { Manifest manifest = Manifest.FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.nodeDataPaths()); if (manifest == null) { manifest = Manifest.empty(); @@ -267,7 +267,7 @@ public void cleanupMetaState(long currentGeneration) { */ public void writeIndexAndUpdateManifest(String reason, IndexMetaData metaData) throws IOException { long generation = writeIndex(reason, metaData); - Manifest manifest = loadManifest(); + Manifest manifest = loadManifestOrEmpty(); Map indices = new HashMap<>(manifest.getIndexGenerations()); indices.put(metaData.getIndex(), generation); manifest = new Manifest(manifest.getGlobalGeneration(), indices); @@ -282,7 +282,7 @@ public void writeIndexAndUpdateManifest(String reason, IndexMetaData metaData) t */ public void writeGlobalStateAndUpdateManifest(String reason, MetaData metaData) throws IOException { long generation = writeGlobalState(reason, metaData); - Manifest manifest = loadManifest(); + Manifest manifest = loadManifestOrEmpty(); manifest = new Manifest(generation, manifest.getIndexGenerations()); long metaStateGeneration = writeManifest(reason, manifest); cleanupGlobalState(generation); From c6ea279254b88e6458e4df20e10a3686421dadc1 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Mon, 12 Nov 2018 13:08:24 +0300 Subject: [PATCH 34/57] Fix bug with missing global generation --- .../cluster/metadata/Manifest.java | 4 ++++ .../elasticsearch/gateway/GatewayMetaState.java | 8 +++----- .../elasticsearch/gateway/MetaStateService.java | 17 ++++++++++++----- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java index fa517ece2062e..252e590c26730 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java @@ -150,6 +150,10 @@ public static Manifest empty() { return new Manifest(MISSING_GLOBAL_GENERATION, Collections.emptyMap()); } + public boolean isGlobalGenerationMissing() { + return globalGeneration == MISSING_GLOBAL_GENERATION; + } + private static final class IndexEntry implements ToXContentFragment { private static final ParseField INDEX_GENERATION_PARSE_FIELD = new ParseField("generation"); private static final ParseField INDEX_PARSE_FIELD = new ParseField("index"); diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index 5817974c35871..b78a454a08513 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -135,11 +135,9 @@ private void upgradeMetaData(MetaDataIndexUpgradeService metaDataIndexUpgradeSer final Manifest newManifest = new Manifest(globalStateGeneration, indices); - if (newManifest.isEmpty() == false) { - final long metaStateGeneration = - metaStateService.writeManifest("startup", newManifest); - cleanupActions.add(() -> metaStateService.cleanupMetaState(metaStateGeneration)); - } + final long metaStateGeneration = + metaStateService.writeManifest("startup", newManifest); + cleanupActions.add(() -> metaStateService.cleanupMetaState(metaStateGeneration)); for (Runnable action : cleanupActions) { action.run(); diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java index bd740e9e39ecf..773412428eb47 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java @@ -67,14 +67,20 @@ Tuple loadFullState() throws IOException { if (manifest.isEmpty()) { return loadFullStateBWC(); } + final MetaData.Builder metaDataBuilder; - final MetaData globalMetaData = MetaData.FORMAT.loadGeneration(logger, namedXContentRegistry, manifest.getGlobalGeneration(), - nodeEnv.nodeDataPaths()); - if (globalMetaData != null) { - metaDataBuilder = MetaData.builder(globalMetaData); + if (manifest.isGlobalGenerationMissing()) { + metaDataBuilder = MetaData.builder(); } else { - throw new IOException("failed to find global metadata [generation: " + manifest.getGlobalGeneration() + "]"); + final MetaData globalMetaData = MetaData.FORMAT.loadGeneration(logger, namedXContentRegistry, manifest.getGlobalGeneration(), + nodeEnv.nodeDataPaths()); + if (globalMetaData != null) { + metaDataBuilder = MetaData.builder(globalMetaData); + } else { + throw new IOException("failed to find global metadata [generation: " + manifest.getGlobalGeneration() + "]"); + } } + for (Map.Entry entry : manifest.getIndexGenerations().entrySet()) { Index index = entry.getKey(); long generation = entry.getValue(); @@ -88,6 +94,7 @@ Tuple loadFullState() throws IOException { ", generation: " + generation + "]"); } } + return new Tuple<>(manifest, metaDataBuilder.build()); } From 95f3aa1fb6f0a18c4662f201a821c0c6f1f13b38 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Mon, 12 Nov 2018 13:57:22 +0300 Subject: [PATCH 35/57] metaState -> manifest rename, and tests for loading empty manifest --- .../gateway/MetaStateServiceTests.java | 66 +++++++++++++------ 1 file changed, 45 insertions(+), 21 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java b/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java index 699fd728606ee..1e1d316cf161b 100644 --- a/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java @@ -32,7 +32,6 @@ import java.io.IOException; import java.util.HashMap; -import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.nullValue; @@ -106,28 +105,53 @@ public void testLoadFullStateBWC() throws Exception { long globalGeneration = metaStateService.writeGlobalState("test_write", metaData); long indexGeneration = metaStateService.writeIndex("test_write", indexMetaData); - Tuple stateAndData = metaStateService.loadFullState(); - Manifest manifest = stateAndData.v1(); + Tuple manifestAndMetaData = metaStateService.loadFullState(); + Manifest manifest = manifestAndMetaData.v1(); assertThat(manifest.getGlobalGeneration(), equalTo(globalGeneration)); assertThat(manifest.getIndexGenerations(), hasKey(indexMetaData.getIndex())); assertThat(manifest.getIndexGenerations().get(indexMetaData.getIndex()), equalTo(indexGeneration)); - MetaData loadedMetaData = stateAndData.v2(); + MetaData loadedMetaData = manifestAndMetaData.v2(); assertThat(loadedMetaData.persistentSettings(), equalTo(metaData.persistentSettings())); assertThat(loadedMetaData.hasIndex("test1"), equalTo(true)); assertThat(loadedMetaData.index("test1"), equalTo(indexMetaData)); } - public void testLoadEmptyState() throws IOException { - Tuple stateAndData = metaStateService.loadFullState(); + public void testLoadEmptyStateNoManifest() throws IOException { + Tuple manifestAndMetaData = metaStateService.loadFullState(); - Manifest manifest = stateAndData.v1(); - assertThat(manifest.getGlobalGeneration(), equalTo(-1L)); - assertThat(manifest.getIndexGenerations().entrySet(), empty()); + Manifest manifest = manifestAndMetaData.v1(); + assertTrue(manifest.isEmpty()); - MetaData metaData = stateAndData.v2(); - MetaData emptyMetaData = MetaData.builder().build(); - assertTrue(MetaData.isGlobalStateEquals(metaData, emptyMetaData)); + MetaData metaData = manifestAndMetaData.v2(); + assertTrue(MetaData.isGlobalStateEquals(metaData, MetaData.EMPTY_META_DATA)); + } + + public void testLoadEmptyStateWithManifest() throws IOException { + Manifest manifest = Manifest.empty(); + metaStateService.writeManifest("test", manifest); + + Tuple manifestAndMetaData = metaStateService.loadFullState(); + assertTrue(manifestAndMetaData.v1().isEmpty()); + MetaData metaData = manifestAndMetaData.v2(); + assertTrue(MetaData.isGlobalStateEquals(metaData, MetaData.EMPTY_META_DATA)); + } + + public void testLoadFullStateMissingGlobalMetaData() throws IOException { + IndexMetaData index = indexMetaData("test1"); + long indexGeneration = metaStateService.writeIndex("test", index); + Manifest manifest = new Manifest(Manifest.empty().getGlobalGeneration(), new HashMap() {{ + put(index.getIndex(), indexGeneration); + }}); + assertTrue(manifest.isGlobalGenerationMissing()); + metaStateService.writeManifest("test", manifest); + + Tuple manifestAndMetaData = metaStateService.loadFullState(); + assertThat(manifestAndMetaData.v1(), equalTo(manifest)); + MetaData loadedMetaData = manifestAndMetaData.v2(); + assertTrue(MetaData.isGlobalStateEquals(loadedMetaData, MetaData.EMPTY_META_DATA)); + assertThat(loadedMetaData.hasIndex("test1"), equalTo(true)); + assertThat(loadedMetaData.index("test1"), equalTo(index)); } public void testLoadFullStateAndUpdate() throws IOException { @@ -144,7 +168,7 @@ public void testLoadFullStateAndUpdate() throws IOException { put(index.getIndex(), indexGeneration); }}); - metaStateService.writeManifest("first meta state write", manifest); + metaStateService.writeManifest("first manifest write", manifest); MetaData newMetaData = MetaData.builder() .persistentSettings(Settings.builder().put("test1", "value2").build()) @@ -152,10 +176,10 @@ public void testLoadFullStateAndUpdate() throws IOException { .build(); globalGeneration = metaStateService.writeGlobalState("second global state write", newMetaData); - Tuple stateAndData = metaStateService.loadFullState(); - assertThat(stateAndData.v1(), equalTo(manifest)); + Tuple manifestAndMetaData = metaStateService.loadFullState(); + assertThat(manifestAndMetaData.v1(), equalTo(manifest)); - MetaData loadedMetaData = stateAndData.v2(); + MetaData loadedMetaData = manifestAndMetaData.v2(); assertThat(loadedMetaData.persistentSettings(), equalTo(metaData.persistentSettings())); assertThat(loadedMetaData.hasIndex("test1"), equalTo(true)); assertThat(loadedMetaData.index("test1"), equalTo(index)); @@ -164,15 +188,15 @@ public void testLoadFullStateAndUpdate() throws IOException { put(index.getIndex(), indexGeneration); }}); - long metaStateGeneration = metaStateService.writeManifest("test", manifest); + long manifestGeneration = metaStateService.writeManifest("second manifest write", manifest); metaStateService.cleanupGlobalState(globalGeneration); metaStateService.cleanupIndex(index.getIndex(), indexGeneration); - metaStateService.cleanupMetaState(metaStateGeneration); + metaStateService.cleanupMetaState(manifestGeneration); - stateAndData = metaStateService.loadFullState(); - assertThat(stateAndData.v1(), equalTo(manifest)); + manifestAndMetaData = metaStateService.loadFullState(); + assertThat(manifestAndMetaData.v1(), equalTo(manifest)); - loadedMetaData = stateAndData.v2(); + loadedMetaData = manifestAndMetaData.v2(); assertThat(loadedMetaData.persistentSettings(), equalTo(newMetaData.persistentSettings())); assertThat(loadedMetaData.hasIndex("test1"), equalTo(true)); assertThat(loadedMetaData.index("test1"), equalTo(index)); From ffd25a64e475073598bb3f3c035746378dd88d60 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Mon, 12 Nov 2018 14:03:43 +0300 Subject: [PATCH 36/57] Revert "Fix tests: FullClusterRestart callback" This reverts commit 87f6ae9 --- .../admin/indices/create/CreateIndexIT.java | 28 ++++++------- .../gateway/GatewayIndexStateIT.java | 41 +++++++------------ .../gateway/QuorumGatewayIT.java | 4 +- .../gateway/RecoveryFromGatewayIT.java | 4 +- .../test/InternalTestCluster.java | 19 +++------ 5 files changed, 35 insertions(+), 61 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java index e88168971f37d..d49ee658c5de1 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java @@ -45,7 +45,6 @@ import java.util.HashMap; import java.util.HashSet; -import java.util.List; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; @@ -351,25 +350,22 @@ public void testIndexWithUnknownSetting() throws Exception { } final IndexMetaData metaData = state.getMetaData().index("test"); - internalCluster().fullRestart(new InternalTestCluster.FullRestartCallback() { - + internalCluster().fullRestart(new InternalTestCluster.RestartCallback() { @Override - public void onAllNodesStopped(List nodeNames) throws Exception { - for (String nodeName : nodeNames) { - if (dataOrMasterNodeNames.contains(nodeName)) { - final MetaStateService metaStateService = internalCluster().getInstance(MetaStateService.class, nodeName); - final IndexMetaData brokenMetaData = - IndexMetaData - .builder(metaData) - .settings(Settings.builder().put(metaData.getSettings()).put("index.foo", true)) - .build(); - // so evil - metaStateService.writeIndexAndUpdateManifest("broken metadata", brokenMetaData); - } + public Settings onNodeStopped(String nodeName) throws Exception { + if (dataOrMasterNodeNames.contains(nodeName)) { + final MetaStateService metaStateService = internalCluster().getInstance(MetaStateService.class, nodeName); + final IndexMetaData brokenMetaData = + IndexMetaData + .builder(metaData) + .settings(Settings.builder().put(metaData.getSettings()).put("index.foo", true)) + .build(); + // so evil + metaStateService.writeIndexAndUpdateManifest("broken metadata", brokenMetaData); } + return Settings.EMPTY; } }); - ensureGreen(metaData.getIndex().getName()); // we have to wait for the index to show up in the metadata or we will fail in a race final ClusterState stateAfterRestart = client().admin().cluster().prepareState().get().getState(); diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java b/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java index 704231907bef5..641dd2ba099d0 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java @@ -48,7 +48,6 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; -import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.InternalTestCluster.RestartCallback; import java.io.IOException; @@ -292,7 +291,7 @@ public void testDanglingIndices() throws Exception { assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(true)); logger.info("--> restarting the nodes"); - internalCluster().fullRestart(new InternalTestCluster.FullRestartCallback() { + internalCluster().fullRestart(new RestartCallback() { @Override public boolean clearData(String nodeName) { return node_1.equals(nodeName); @@ -460,18 +459,12 @@ public void testRecoverMissingAnalyzer() throws Exception { } ClusterState state = client().admin().cluster().prepareState().get().getState(); IndexMetaData metaData = state.getMetaData().index("test"); - - internalCluster().fullRestart(new InternalTestCluster.FullRestartCallback(){ - @Override - public void onAllNodesStopped(List nodeNames) throws Exception { - for (MetaStateService metaStateService : internalCluster().getInstances(MetaStateService.class)) { - IndexMetaData brokenMeta = IndexMetaData.builder(metaData).settings(metaData.getSettings() - .filter((s) -> "index.analysis.analyzer.test.tokenizer".equals(s) == false)).build(); - metaStateService.writeIndexAndUpdateManifest("broken metadata", brokenMeta); - } - } - }); - + for (MetaStateService metaStateService : internalCluster().getInstances(MetaStateService.class)) { + IndexMetaData brokenMeta = IndexMetaData.builder(metaData).settings(metaData.getSettings() + .filter((s) -> "index.analysis.analyzer.test.tokenizer".equals(s) == false)).build(); + metaStateService.writeIndexAndUpdateManifest("broken metadata", brokenMeta); + } + internalCluster().fullRestart(); // ensureGreen(closedIndex) waits for the index to show up in the metadata // this is crucial otherwise the state call below might not contain the index yet ensureGreen(metaData.getIndex().getName()); @@ -503,19 +496,13 @@ public void testArchiveBrokenClusterSettings() throws Exception { } ClusterState state = client().admin().cluster().prepareState().get().getState(); MetaData metaData = state.getMetaData(); - - internalCluster().fullRestart(new InternalTestCluster.FullRestartCallback(){ - @Override - public void onAllNodesStopped(List nodeNames) throws Exception { - for (MetaStateService metaStateService : internalCluster().getInstances(MetaStateService.class)) { - MetaData brokenMeta = MetaData.builder(metaData).persistentSettings(Settings.builder() - .put(metaData.persistentSettings()).put("this.is.unknown", true) - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), "broken").build()).build(); - metaStateService.writeGlobalStateAndUpdateManifest("broken metadata", brokenMeta); - } - } - }); - + for (MetaStateService metaStateService : internalCluster().getInstances(MetaStateService.class)) { + MetaData brokenMeta = MetaData.builder(metaData).persistentSettings(Settings.builder() + .put(metaData.persistentSettings()).put("this.is.unknown", true) + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), "broken").build()).build(); + metaStateService.writeGlobalStateAndUpdateManifest("broken metadata", brokenMeta); + } + internalCluster().fullRestart(); ensureYellow("test"); // wait for state recovery state = client().admin().cluster().prepareState().get().getState(); assertEquals("true", state.metaData().persistentSettings().get("archived.this.is.unknown")); diff --git a/server/src/test/java/org/elasticsearch/gateway/QuorumGatewayIT.java b/server/src/test/java/org/elasticsearch/gateway/QuorumGatewayIT.java index c31b095b32323..b16e2e2f6c505 100644 --- a/server/src/test/java/org/elasticsearch/gateway/QuorumGatewayIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/QuorumGatewayIT.java @@ -26,7 +26,7 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; -import org.elasticsearch.test.InternalTestCluster; +import org.elasticsearch.test.InternalTestCluster.RestartCallback; import java.util.concurrent.TimeUnit; @@ -64,7 +64,7 @@ public void testQuorumRecovery() throws Exception { assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).get(), 2L); } logger.info("--> restart all nodes"); - internalCluster().fullRestart(new InternalTestCluster.FullRestartCallback() { + internalCluster().fullRestart(new RestartCallback() { @Override public Settings onNodeStopped(String nodeName) throws Exception { return null; diff --git a/server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java b/server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java index 43ba81f91f6d3..a8f2cfab2b79b 100644 --- a/server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java @@ -300,7 +300,7 @@ public void testTwoNodeFirstNodeCleared() throws Exception { Map primaryTerms = assertAndCapturePrimaryTerms(null); - internalCluster().fullRestart(new InternalTestCluster.FullRestartCallback() { + internalCluster().fullRestart(new RestartCallback() { @Override public Settings onNodeStopped(String nodeName) throws Exception { return Settings.builder().put("gateway.recover_after_nodes", 2).build(); @@ -549,7 +549,7 @@ public void testStartedShardFoundIfStateNotYetProcessed() throws Exception { final boolean corrupt = randomBoolean(); - internalCluster().fullRestart(new InternalTestCluster.FullRestartCallback() { + internalCluster().fullRestart(new RestartCallback() { @Override public Settings onNodeStopped(String nodeName) throws Exception { // make sure state is not recovered diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index 0eec82e8d00a9..718e4928d6cd4 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -1687,7 +1687,7 @@ public Settings onNodeStopped(String node) { * Restarts all nodes in the cluster. It first stops all nodes and then restarts all the nodes again. */ public void fullRestart() throws Exception { - fullRestart(new FullRestartCallback()); + fullRestart(EMPTY_CALLBACK); } /** @@ -1731,7 +1731,7 @@ private void restartNode(NodeAndClient nodeAndClient, RestartCallback callback) /** * Restarts all nodes in the cluster. It first stops all nodes and then restarts all the nodes again. */ - public synchronized void fullRestart(FullRestartCallback callback) throws Exception { + public synchronized void fullRestart(RestartCallback callback) throws Exception { int numNodesRestarted = 0; Map, List> nodesByRoles = new HashMap<>(); Set[] rolesOrderedByOriginalStartupOrder = new Set[nextNodeId.get()]; @@ -1749,8 +1749,6 @@ public synchronized void fullRestart(FullRestartCallback callback) throws Except nodesByRoles.computeIfAbsent(discoveryNode.getRoles(), k -> new ArrayList<>()).add(nodeAndClient); } - callback.onAllNodesStopped(nodes.values().stream().map(nodeAndClient -> nodeAndClient.name).collect(Collectors.toList())); - assert nodesByRoles.values().stream().collect(Collectors.summingInt(List::size)) == nodes.size(); // randomize start up order, but making sure that: @@ -2205,18 +2203,11 @@ public boolean test(Settings settings) { } } - /** - * A class that is called during {@link #fullRestart(FullRestartCallback)} - * to execute actions at certain stages of the restart. - */ - public static class FullRestartCallback extends RestartCallback { - public void onAllNodesStopped(List nodeNames) throws Exception { - } - } /** - * A class that is called during {@link #rollingRestart(InternalTestCluster.RestartCallback)} - * to execute actions at certain stages of the restart. + * An abstract class that is called during {@link #rollingRestart(InternalTestCluster.RestartCallback)} + * and / or {@link #fullRestart(InternalTestCluster.RestartCallback)} to execute actions at certain + * stages of the restart. */ public static class RestartCallback { From f21ddf9538bc0750f3ec6911039c058013243412 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Mon, 12 Nov 2018 14:42:37 +0300 Subject: [PATCH 37/57] fullRestart should call onNodeStopped callback, before re-creating nodes --- .../test/InternalTestCluster.java | 38 ++++++++++++------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index 718e4928d6cd4..de796fdc39baf 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -923,18 +923,9 @@ void restart(RestartCallback callback, boolean clearDataIfNeeded, int minMasterN if (!node.isClosed()) { closeNode(); } - recreateNodeOnRestart(callback, clearDataIfNeeded, minMasterNodes, () -> rebuildUnicastHostFiles(emptyList())); - startNode(); - } - - /** - * rebuilds a new node object using the current node settings and starts it - */ - void recreateNodeOnRestart(RestartCallback callback, boolean clearDataIfNeeded, int minMasterNodes, - Runnable onTransportServiceStarted) throws Exception { assert callback != null; Settings callbackSettings = callback.onNodeStopped(name); - Settings.Builder newSettings = Settings.builder(); + Builder newSettings = Settings.builder(); if (callbackSettings != null) { newSettings.put(callbackSettings); } @@ -945,9 +936,10 @@ void recreateNodeOnRestart(RestartCallback callback, boolean clearDataIfNeeded, if (clearDataIfNeeded) { clearDataIfNeeded(callback); } - createNewNode(newSettings.build(), onTransportServiceStarted); + createNewNode(newSettings.build(), () -> rebuildUnicastHostFiles(emptyList())); // make sure cached client points to new node resetClient(); + startNode(); } private void clearDataIfNeeded(RestartCallback callback) throws IOException { @@ -1769,11 +1761,29 @@ public synchronized void fullRestart(RestartCallback callback) throws Exception } assert nodesByRoles.values().stream().collect(Collectors.summingInt(List::size)) == 0; + //Let's call onNodeStopped callback on all nodes, before re-creating the nodes + final Map newNodeSettings = new HashMap<>(); + for (NodeAndClient nodeAndClient : startUpOrder) { + Settings callbackSettings = callback.onNodeStopped(nodeAndClient.name); + Builder newSettings = Settings.builder(); + if (callbackSettings != null) { + newSettings.put(callbackSettings); + } + if (autoManageMinMasterNodes) { + int minMasterNodes = getMinMasterNodes(getMasterNodesCount()); + assert DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(newSettings.build()) == false : "min master nodes is auto managed"; + newSettings.put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minMasterNodes).build(); + } + newNodeSettings.put(nodeAndClient.name, newSettings.build()); + } + + //Now we can re-create the nodes for (NodeAndClient nodeAndClient : startUpOrder) { logger.info("resetting node [{}] ", nodeAndClient.name); - // we already cleared data folders, before starting nodes up - nodeAndClient.recreateNodeOnRestart(callback, false, autoManageMinMasterNodes ? getMinMasterNodes(getMasterNodesCount()) : -1, - () -> rebuildUnicastHostFiles(startUpOrder)); + + nodeAndClient.createNewNode(newNodeSettings.get(nodeAndClient.name), () -> rebuildUnicastHostFiles(startUpOrder)); + // make sure cached client points to new node + nodeAndClient.resetClient(); } startAndPublishNodesAndClients(startUpOrder); From f1ff7325f3b809d678e8cfa58641aec8f6c73eba Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Mon, 12 Nov 2018 14:43:03 +0300 Subject: [PATCH 38/57] Fix broken meta tests --- .../admin/indices/create/CreateIndexIT.java | 2 +- .../gateway/GatewayIndexStateIT.java | 59 ++++++++++++------- 2 files changed, 39 insertions(+), 22 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java index d49ee658c5de1..88a2cdf6c0675 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java @@ -363,7 +363,7 @@ public Settings onNodeStopped(String nodeName) throws Exception { // so evil metaStateService.writeIndexAndUpdateManifest("broken metadata", brokenMetaData); } - return Settings.EMPTY; + return super.onNodeStopped(nodeName); } }); ensureGreen(metaData.getIndex().getName()); // we have to wait for the index to show up in the metadata or we will fail in a race diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java b/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java index 641dd2ba099d0..414f37799678d 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java @@ -395,19 +395,24 @@ public void testRecoverBrokenIndexMetadata() throws Exception { .waitForNoRelocatingShards(true).waitForNodes("2")).actionGet(); } ClusterState state = client().admin().cluster().prepareState().get().getState(); - IndexMetaData metaData = state.getMetaData().index("test"); - for (MetaStateService metaStateService : internalCluster().getInstances(MetaStateService.class)) { - IndexMetaData brokenMeta = IndexMetaData.builder(metaData).settings(Settings.builder().put(metaData.getSettings()) + + final IndexMetaData metaData = state.getMetaData().index("test"); + final IndexMetaData brokenMeta = IndexMetaData.builder(metaData).settings(Settings.builder().put(metaData.getSettings()) .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT.minimumIndexCompatibilityVersion().id) - // this is invalid but should be archived + // this is invalid but should be archived .put("index.similarity.BM25.type", "classic") - // this one is not validated ahead of time and breaks allocation + // this one is not validated ahead of time and breaks allocation .put("index.analysis.filter.myCollator.type", "icu_collation") - ).build(); + ).build(); + internalCluster().fullRestart(new RestartCallback(){ + @Override + public Settings onNodeStopped(String nodeName) throws Exception { + final MetaStateService metaStateService = internalCluster().getInstance(MetaStateService.class, nodeName); + metaStateService.writeIndexAndUpdateManifest("broken metadata", brokenMeta); + return super.onNodeStopped(nodeName); + } + }); - metaStateService.writeIndexAndUpdateManifest("broken metadata", brokenMeta); - } - internalCluster().fullRestart(); // ensureGreen(closedIndex) waits for the index to show up in the metadata // this is crucial otherwise the state call below might not contain the index yet ensureGreen(metaData.getIndex().getName()); @@ -458,13 +463,19 @@ public void testRecoverMissingAnalyzer() throws Exception { .waitForNoRelocatingShards(true).waitForNodes("2")).actionGet(); } ClusterState state = client().admin().cluster().prepareState().get().getState(); - IndexMetaData metaData = state.getMetaData().index("test"); - for (MetaStateService metaStateService : internalCluster().getInstances(MetaStateService.class)) { - IndexMetaData brokenMeta = IndexMetaData.builder(metaData).settings(metaData.getSettings() + + final IndexMetaData metaData = state.getMetaData().index("test"); + final IndexMetaData brokenMeta = IndexMetaData.builder(metaData).settings(metaData.getSettings() .filter((s) -> "index.analysis.analyzer.test.tokenizer".equals(s) == false)).build(); - metaStateService.writeIndexAndUpdateManifest("broken metadata", brokenMeta); - } - internalCluster().fullRestart(); + internalCluster().fullRestart(new RestartCallback(){ + @Override + public Settings onNodeStopped(String nodeName) throws Exception { + final MetaStateService metaStateService = internalCluster().getInstance(MetaStateService.class, nodeName); + metaStateService.writeIndexAndUpdateManifest("broken metadata", brokenMeta); + return super.onNodeStopped(nodeName); + } + }); + // ensureGreen(closedIndex) waits for the index to show up in the metadata // this is crucial otherwise the state call below might not contain the index yet ensureGreen(metaData.getIndex().getName()); @@ -495,14 +506,20 @@ public void testArchiveBrokenClusterSettings() throws Exception { .waitForNoRelocatingShards(true).waitForNodes("2")).actionGet(); } ClusterState state = client().admin().cluster().prepareState().get().getState(); - MetaData metaData = state.getMetaData(); - for (MetaStateService metaStateService : internalCluster().getInstances(MetaStateService.class)) { - MetaData brokenMeta = MetaData.builder(metaData).persistentSettings(Settings.builder() + + final MetaData metaData = state.getMetaData(); + final MetaData brokenMeta = MetaData.builder(metaData).persistentSettings(Settings.builder() .put(metaData.persistentSettings()).put("this.is.unknown", true) .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), "broken").build()).build(); - metaStateService.writeGlobalStateAndUpdateManifest("broken metadata", brokenMeta); - } - internalCluster().fullRestart(); + internalCluster().fullRestart(new RestartCallback(){ + @Override + public Settings onNodeStopped(String nodeName) throws Exception { + final MetaStateService metaStateService = internalCluster().getInstance(MetaStateService.class, nodeName); + metaStateService.writeGlobalStateAndUpdateManifest("broken metadata", brokenMeta); + return super.onNodeStopped(nodeName); + } + }); + ensureYellow("test"); // wait for state recovery state = client().admin().cluster().prepareState().get().getState(); assertEquals("true", state.metaData().persistentSettings().get("archived.this.is.unknown")); From fb3fc72fc84739a4572d42d77ecc27d8b57b1891 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Mon, 12 Nov 2018 17:13:34 +0300 Subject: [PATCH 39/57] write method should perform cleanup if it fails, s/stateId/generationId --- .../gateway/MetaDataStateFormat.java | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java b/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java index 7f9eacf89b804..a3eba3d99ea5c 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java @@ -186,7 +186,6 @@ public final long writeAndCleanup(final T state, final Path... locations) throws return currentGeneration; } - /** * Writes the given state to the given directories. The state is written to a * state directory ({@value #STATE_DIR_NAME}) underneath each of the given file locations and is created if it @@ -194,7 +193,10 @@ public final long writeAndCleanup(final T state, final Path... locations) throws * it's target filename of the pattern {@code {prefix}{version}.st}. * If this method returns without exception there is a guarantee that state is persisted to the disk and loadLatestState will return * it.
- * This method does not perform cleanup of old state files. + * This method always performs cleanup of temporary files regardless whether it succeeds or fails. Cleanup logic for state files is + * more involved. + * If this method fails with an exception, it performs cleanup of newly created state file. + * But if this method succeeds, it does not perform cleanup of old state files. * If this write succeeds, but some further write fails, you may want to rollback the transaction and keep old file around. * After transaction is finished use {@link #cleanupOldFiles(long, Path[])} for the clean-up. * If this write is not a part of bigger transaction, consider using {@link #writeAndCleanup(Object, Path...)} method instead. @@ -212,15 +214,16 @@ public final long write(final T state, final Path... locations) throws WriteStat throw new IllegalArgumentException("One or more locations required"); } - long maxStateId; + final long oldGenerationId, newGenerationId; try { - maxStateId = findMaxStateId(prefix, locations) + 1; + oldGenerationId = findMaxGenerationId(prefix, locations); + newGenerationId = oldGenerationId + 1; } catch (Exception e) { - throw new WriteStateException(false, "exception during looking up max generation id", e); + throw new WriteStateException(false, "exception during looking up new generation id", e); } - assert maxStateId >= 0 : "maxStateId must be positive but was: [" + maxStateId + "]"; + assert newGenerationId >= 0 : "newGenerationId must be positive but was: [" + oldGenerationId + "]"; - final String fileName = getStateFileName(maxStateId); + final String fileName = getStateFileName(newGenerationId); final String tmpFileName = fileName + ".tmp"; List> directories = new ArrayList<>(); @@ -238,6 +241,9 @@ public final long write(final T state, final Path... locations) throws WriteStat copyStateToExtraLocations(directories, tmpFileName); performRenames(tmpFileName, fileName, directories); performStateDirectoriesFsync(directories); + } catch (WriteStateException e) { + cleanupOldFiles(oldGenerationId, locations); + throw e; } finally { for (Tuple pathAndDirectory : directories) { deleteFileIgnoreExceptions(pathAndDirectory.v1(), pathAndDirectory.v2(), tmpFileName); @@ -245,7 +251,7 @@ public final long write(final T state, final Path... locations) throws WriteStat } } - return maxStateId; + return newGenerationId; } protected XContentBuilder newXContentBuilder(XContentType type, OutputStream stream ) throws IOException { @@ -330,7 +336,7 @@ public void cleanupOldFiles(final long currentGeneration, Path[] locations) { * @return maximum id of state file or -1 if no such files are found * @throws IOException if IOException occurs */ - private long findMaxStateId(final String prefix, Path... locations) throws IOException { + private long findMaxGenerationId(final String prefix, Path... locations) throws IOException { long maxId = -1; for (Path dataLocation : locations) { final Path resolve = dataLocation.resolve(STATE_DIR_NAME); @@ -405,7 +411,6 @@ public T loadGeneration(Logger logger, NamedXContentRegistry namedXContentRegist return null; } - /** * Tries to load the latest state from the given data-locations. * @@ -415,12 +420,12 @@ public T loadGeneration(Logger logger, NamedXContentRegistry namedXContentRegist */ public Tuple loadLatestStateWithGeneration(Logger logger, NamedXContentRegistry namedXContentRegistry, Path... dataLocations) throws IOException { - long generation = findMaxStateId(prefix, dataLocations); + long generation = findMaxGenerationId(prefix, dataLocations); T state = loadGeneration(logger, namedXContentRegistry, generation, dataLocations); if (generation > -1 && state == null) { throw new IllegalStateException("unable to find state files with generation id " + generation + - " returned by findMaxStateId function, in data folders [" + + " returned by findMaxGenerationId function, in data folders [" + Arrays.stream(dataLocations).map(Path::toAbsolutePath). map(Object::toString).collect(Collectors.joining(", ")) + "], concurrent writes?"); From 20c9813bf637fe7a8c0b22e2078b098b3240ee4e Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Mon, 12 Nov 2018 18:28:28 +0300 Subject: [PATCH 40/57] GatewayMetaState perform cleanup if apply state fails --- .../gateway/GatewayMetaState.java | 150 ++++++++++++------ .../gateway/IndexMetaDataWriter.java | 28 ---- .../gateway/MetaStateService.java | 8 +- .../gateway/GatewayMetaStateTests.java | 55 +++---- .../gateway/MetaStateServiceTests.java | 2 +- 5 files changed, 127 insertions(+), 116 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/gateway/IndexMetaDataWriter.java diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index b78a454a08513..1338e5606c7ef 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -113,13 +113,12 @@ private void upgradeMetaData(MetaDataIndexUpgradeService metaDataIndexUpgradeSer // if there is manifest file, it means metadata is properly persisted to all data paths // if there is no manifest file (upgrade from 6.x to 7.x) metadata might be missing on some data paths, // but anyway we will re-write it as soon as we receive first ClusterState - List cleanupActions = new ArrayList<>(); + final Transaction tx = new Transaction(metaStateService, manifest); final MetaData upgradedMetaData = upgradeMetaData(metaData, metaDataIndexUpgradeService, metaDataUpgrader); final long globalStateGeneration; if (MetaData.isGlobalStateEquals(metaData, upgradedMetaData) == false) { - globalStateGeneration = metaStateService.writeGlobalState("upgrade", upgradedMetaData); - cleanupActions.add(() -> metaStateService.cleanupGlobalState(globalStateGeneration)); + globalStateGeneration = tx.writeGlobalState("upgrade", upgradedMetaData); } else { globalStateGeneration = manifest.getGlobalGeneration(); } @@ -127,21 +126,13 @@ private void upgradeMetaData(MetaDataIndexUpgradeService metaDataIndexUpgradeSer Map indices = new HashMap<>(manifest.getIndexGenerations()); for (IndexMetaData indexMetaData : upgradedMetaData) { if (metaData.hasIndexMetaData(indexMetaData) == false) { - final long generation = metaStateService.writeIndex("upgrade", indexMetaData); - cleanupActions.add(() -> metaStateService.cleanupIndex(indexMetaData.getIndex(), generation)); + final long generation = tx.writeIndex("upgrade", indexMetaData); indices.put(indexMetaData.getIndex(), generation); } } final Manifest newManifest = new Manifest(globalStateGeneration, indices); - - final long metaStateGeneration = - metaStateService.writeManifest("startup", newManifest); - cleanupActions.add(() -> metaStateService.cleanupMetaState(metaStateGeneration)); - - for (Runnable action : cleanupActions) { - action.run(); - } + tx.writeManifest("startup", newManifest); } catch (Exception e) { logger.error("failed to read or re-write local state, exiting...", e); throw e; @@ -177,12 +168,92 @@ public void applyClusterState(ClusterChangedEvent event) { } try { + if (previousManifest == null) { + previousManifest = metaStateService.loadManifestOrEmpty(); + } updateMetaData(event); } catch (Exception e) { logger.warn("Exception occurred when storing new meta data", e); } } + /** + * This class is used to write changed global {@link MetaData}, {@link IndexMetaData} and {@link Manifest} to disk. + * This class delegates write* calls to corresponding write calls in {@link MetaStateService} and + * additionally it keeps track of cleanup actions to be performed if transaction succeeds or fails. + * Transaction succeeds if {@link #writeManifest(String, Manifest)} call succeeds, transaction fails if + * any write* call fails. + * Once transaction succeeds/fails it can no longer be used. + */ + static class Transaction { + private static final String TRANSACTION_COMPLETED_MSG = "Transaction is already completed"; + private final List commitCleanupActions; + private final List rollbackCleanupActions; + private final Manifest previousManifest; + private final MetaStateService metaStateService; + private boolean finished; + + Transaction(MetaStateService metaStateService, Manifest previousManifest) { + this.metaStateService = metaStateService; + assert previousManifest != null; + this.previousManifest = previousManifest; + this.commitCleanupActions = new ArrayList<>(); + this.rollbackCleanupActions = new ArrayList<>(); + this.finished = false; + } + + long writeGlobalState(String reason, MetaData metaData) throws WriteStateException { + assert finished == false : TRANSACTION_COMPLETED_MSG; + try { + long generation = metaStateService.writeGlobalState(reason, metaData); + commitCleanupActions.add(() -> metaStateService.cleanupGlobalState(generation)); + if (previousManifest.isGlobalGenerationMissing() == false) { + rollbackCleanupActions.add(() -> metaStateService.cleanupGlobalState(previousManifest.getGlobalGeneration())); + } + return generation; + } catch (WriteStateException e) { + rollback(); + throw e; + } + } + + long writeIndex(String reason, IndexMetaData metaData) throws WriteStateException { + assert finished == false : TRANSACTION_COMPLETED_MSG; + try { + Index index = metaData.getIndex(); + long generation = metaStateService.writeIndex(reason, metaData); + commitCleanupActions.add(() -> metaStateService.cleanupIndex(index, generation)); + Long previousGeneration = previousManifest.getIndexGenerations().get(index); + if (previousGeneration != null) { + rollbackCleanupActions.add(() -> metaStateService.cleanupIndex(index, previousGeneration)); + } + return generation; + } catch (WriteStateException e) { + rollback(); + throw e; + } + } + + long writeManifest(String reason, Manifest manifest) throws WriteStateException { + assert finished == false : TRANSACTION_COMPLETED_MSG; + try { + long generation = metaStateService.writeManifest(reason, manifest); + commitCleanupActions.add(() -> metaStateService.cleanupManifest(generation)); + commitCleanupActions.forEach(Runnable::run); + finished = true; + return generation; + } catch (WriteStateException e) { + rollback(); + throw e; + } + } + + void rollback() { + rollbackCleanupActions.forEach(Runnable::run); + finished = true; + } + } + /** * Updates meta state and meta data on disk according to {@link ClusterChangedEvent}. * @@ -194,32 +265,24 @@ private void updateMetaData(ClusterChangedEvent event) throws IOException { ClusterState previousState = event.previousState(); MetaData newMetaData = newState.metaData(); - List cleanupActions = new ArrayList<>(); - long globalStateGeneration = writeGlobalState(newMetaData, cleanupActions); - Map newIndices = writeIndicesMetadata(newState, previousState, cleanupActions); - Manifest manifest = new Manifest(globalStateGeneration, newIndices); - writeManifest(manifest, cleanupActions); - - for (Runnable action : cleanupActions) { - action.run(); - } + final Transaction tx = new Transaction(metaStateService, previousManifest); + long globalStateGeneration = writeGlobalState(tx, newMetaData); + Map indexGenerations = writeIndicesMetadata(tx, newState, previousState); + Manifest manifest = new Manifest(globalStateGeneration, indexGenerations); + writeManifest(tx, manifest); previousMetaData = newMetaData; previousManifest = manifest; } - private void writeManifest(Manifest manifest, List cleanupActions) throws IOException { + private void writeManifest(Transaction tx, Manifest manifest) throws IOException { if (manifest.equals(previousManifest) == false) { - long generation = metaStateService.writeManifest("changed", manifest); - cleanupActions.add(() -> metaStateService.cleanupMetaState(generation)); + tx.writeManifest("changed", manifest); } } - private Map writeIndicesMetadata(ClusterState newState, ClusterState previousState, List cleanupActions) + private Map writeIndicesMetadata(Transaction tx, ClusterState newState, ClusterState previousState) throws IOException { - if (previousManifest == null) { - previousManifest = metaStateService.loadManifestOrEmpty(); - } Map previouslyWrittenIndices = previousManifest.getIndexGenerations(); Set relevantIndices = getRelevantIndices(newState, previousState, previouslyWrittenIndices.keySet()); @@ -229,18 +292,16 @@ private Map writeIndicesMetadata(ClusterState newState, ClusterStat newState.metaData()); for (IndexMetaDataAction action : actions) { - long generation = action.execute(metaStateService, cleanupActions); + long generation = action.execute(tx); newIndices.put(action.getIndex(), generation); } return newIndices; } - private long writeGlobalState(MetaData newMetaData, List cleanupActions) throws IOException { + private long writeGlobalState(Transaction tx, MetaData newMetaData) throws IOException { if (previousMetaData == null || MetaData.isGlobalStateEquals(previousMetaData, newMetaData) == false) { - long generation = metaStateService.writeGlobalState("changed", newMetaData); - cleanupActions.add(() -> metaStateService.cleanupGlobalState(generation)); - return generation; + return tx.writeGlobalState("changed", newMetaData); } return previousManifest.getGlobalGeneration(); } @@ -458,15 +519,12 @@ public interface IndexMetaDataAction { Index getIndex(); /** - * Executes this action using writer and potentially adding cleanup action to the list of cleanupActions. + * Executes this action using provided {@link Transaction}. * - * @param writer entity that can write index metadata to disk and perform cleanup afterwards. We prefer - * {@link IndexMetaDataWriter} interface in place of {@link MetaStateService} for easier unit testing. - * @param cleanupActions list of actions, which is expected to be mutated by adding new clean up action to it. * @return new index metadata state generation, to be used in manifest file. * @throws WriteStateException if exception occurs. */ - long execute(IndexMetaDataWriter writer, List cleanupActions) throws WriteStateException; + long execute(Transaction tx) throws WriteStateException; } public static class KeepPreviousGeneration implements IndexMetaDataAction { @@ -484,7 +542,7 @@ public Index getIndex() { } @Override - public long execute(IndexMetaDataWriter writer, List cleanupActions) { + public long execute(Transaction tx) { return generation; } } @@ -502,10 +560,8 @@ public Index getIndex() { } @Override - public long execute(IndexMetaDataWriter writer, List cleanupActions) throws WriteStateException { - long generation = writer.writeIndex("freshly created", indexMetaData); - cleanupActions.add(() -> writer.cleanupIndex(indexMetaData.getIndex(), generation)); - return generation; + public long execute(Transaction tx) throws WriteStateException { + return tx.writeIndex("freshly created", indexMetaData); } } @@ -524,12 +580,10 @@ public Index getIndex() { } @Override - public long execute(IndexMetaDataWriter writer, List cleanupActions) throws WriteStateException { - long generation = writer.writeIndex( + public long execute(Transaction tx) throws WriteStateException { + return tx.writeIndex( "version changed from [" + oldIndexMetaData.getVersion() + "] to [" + newIndexMetaData.getVersion() + "]", newIndexMetaData); - cleanupActions.add(() -> writer.cleanupIndex(newIndexMetaData.getIndex(), generation)); - return generation; } } } diff --git a/server/src/main/java/org/elasticsearch/gateway/IndexMetaDataWriter.java b/server/src/main/java/org/elasticsearch/gateway/IndexMetaDataWriter.java deleted file mode 100644 index 45d6235ef966d..0000000000000 --- a/server/src/main/java/org/elasticsearch/gateway/IndexMetaDataWriter.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.gateway; - -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.index.Index; - -public interface IndexMetaDataWriter { - long writeIndex(String reason, IndexMetaData indexMetaData) throws WriteStateException; - void cleanupIndex(Index index, long currentGeneration); -} diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java index 773412428eb47..93470e2bbb0ee 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java @@ -41,7 +41,7 @@ /** * Handles writing and loading {@link Manifest}, {@link MetaData} and {@link IndexMetaData} */ -public class MetaStateService extends AbstractComponent implements IndexMetaDataWriter { +public class MetaStateService extends AbstractComponent { private final NodeEnvironment nodeEnv; private final NamedXContentRegistry namedXContentRegistry; @@ -264,7 +264,7 @@ public void cleanupIndex(Index index, long currentGeneration) { * * @param currentGeneration current state generation to keep in the directory. */ - public void cleanupMetaState(long currentGeneration) { + public void cleanupManifest(long currentGeneration) { Manifest.FORMAT.cleanupOldFiles(currentGeneration, nodeEnv.nodeDataPaths()); } @@ -280,7 +280,7 @@ public void writeIndexAndUpdateManifest(String reason, IndexMetaData metaData) t manifest = new Manifest(manifest.getGlobalGeneration(), indices); long metaStateGeneration = writeManifest(reason, manifest); cleanupIndex(metaData.getIndex(), generation); - cleanupMetaState(metaStateGeneration); + cleanupManifest(metaStateGeneration); } /** @@ -293,6 +293,6 @@ public void writeGlobalStateAndUpdateManifest(String reason, MetaData metaData) manifest = new Manifest(generation, manifest.getIndexGenerations()); long metaStateGeneration = writeManifest(reason, manifest); cleanupGlobalState(generation); - cleanupMetaState(metaStateGeneration); + cleanupManifest(metaStateGeneration); } } \ No newline at end of file diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java index 2077025ac5148..937849d14e04d 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java @@ -35,6 +35,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.plugins.MetaDataUpgrader; import org.elasticsearch.test.TestCustomMetaData; +import org.mockito.ArgumentCaptor; import java.util.ArrayList; import java.util.Arrays; @@ -49,6 +50,12 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; public class GatewayMetaStateTests extends ESAllocationTestCase { @@ -246,49 +253,27 @@ public void testResolveStatesToBeWritten() throws WriteStateException { for (GatewayMetaState.IndexMetaDataAction action : actions) { if (action instanceof GatewayMetaState.KeepPreviousGeneration) { assertThat(action.getIndex(), equalTo(notChangedIndex.getIndex())); - assertThat(action.execute(null, cleanupActions), equalTo(3L)); + GatewayMetaState.Transaction tx = mock(GatewayMetaState.Transaction.class); + assertThat(action.execute(tx), equalTo(3L)); + verifyZeroInteractions(tx); } if (action instanceof GatewayMetaState.WriteNewIndexMetaData) { assertThat(action.getIndex(), equalTo(newIndex.getIndex())); - action.execute(new IndexMetaDataWriter() { - @Override - public long writeIndex(String reason, IndexMetaData indexMetaData) { - assertThat(reason, equalTo("freshly created")); - assertThat(indexMetaData, equalTo(newIndex)); - return 0L; - } - - @Override - public void cleanupIndex(Index index, long currentGeneration) { - assertThat(index, equalTo(newIndex.getIndex())); - assertThat(currentGeneration, equalTo(0L)); - } - }, cleanupActions); + GatewayMetaState.Transaction tx = mock(GatewayMetaState.Transaction.class); + when(tx.writeIndex("freshly created", newIndex)).thenReturn(0L); + assertThat(action.execute(tx), equalTo(0L)); } if (action instanceof GatewayMetaState.WriteChangedIndexMetaData) { assertThat(action.getIndex(), equalTo(newVersionChangedIndex.getIndex())); - action.execute(new IndexMetaDataWriter() { - @Override - public long writeIndex(String reason, IndexMetaData indexMetaData) { - assertThat(reason, containsString(Long.toString(versionChangedIndex.getVersion()))); - assertThat(reason, containsString(Long.toString(newVersionChangedIndex.getVersion()))); - assertThat(indexMetaData, equalTo(newVersionChangedIndex)); - return 3L; - } - - @Override - public void cleanupIndex(Index index, long currentGeneration) { - assertThat(index, equalTo(newVersionChangedIndex.getIndex())); - assertThat(currentGeneration, equalTo(3L)); - } - }, cleanupActions); + GatewayMetaState.Transaction tx = mock(GatewayMetaState.Transaction.class); + when(tx.writeIndex(anyString(), eq(newVersionChangedIndex))).thenReturn(3L); + assertThat(action.execute(tx), equalTo(3L)); + ArgumentCaptor reason = ArgumentCaptor.forClass(String.class); + verify(tx).writeIndex(reason.capture(), eq(newVersionChangedIndex)); + assertThat(reason.getValue(), containsString(Long.toString(versionChangedIndex.getVersion()))); + assertThat(reason.getValue(), containsString(Long.toString(newVersionChangedIndex.getVersion()))); } } - - assertThat(cleanupActions, hasSize(2)); - for (Runnable cleanupAction : cleanupActions) { - cleanupAction.run(); - } } public void testAddCustomMetaDataOnUpgrade() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java b/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java index 1e1d316cf161b..0bae494e7a9c2 100644 --- a/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java @@ -191,7 +191,7 @@ public void testLoadFullStateAndUpdate() throws IOException { long manifestGeneration = metaStateService.writeManifest("second manifest write", manifest); metaStateService.cleanupGlobalState(globalGeneration); metaStateService.cleanupIndex(index.getIndex(), indexGeneration); - metaStateService.cleanupMetaState(manifestGeneration); + metaStateService.cleanupManifest(manifestGeneration); manifestAndMetaData = metaStateService.loadFullState(); assertThat(manifestAndMetaData.v1(), equalTo(manifest)); From fb56acc4ca86a6bc0740aeed151fe52d0d7548da Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Mon, 12 Nov 2018 19:32:44 +0300 Subject: [PATCH 41/57] Remove unused import --- .../main/java/org/elasticsearch/gateway/GatewayMetaState.java | 1 - .../main/java/org/elasticsearch/gateway/MetaStateService.java | 1 - 2 files changed, 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index 1baaac1b5e98e..275edc7205aff 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -36,7 +36,6 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.env.NodeEnvironment; diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java index 1e46ccc9851d9..8397b0c0536d1 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java @@ -27,7 +27,6 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.Index; From bbab85a6410816650e3b73423678e81aef420b8e Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Tue, 13 Nov 2018 09:46:40 +0300 Subject: [PATCH 42/57] Always cleanup old state on rollback, even if prev gen is missing --- .../org/elasticsearch/gateway/GatewayMetaState.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index 275edc7205aff..db5d54f5fae51 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -210,9 +210,7 @@ long writeGlobalState(String reason, MetaData metaData) throws WriteStateExcepti try { long generation = metaStateService.writeGlobalState(reason, metaData); commitCleanupActions.add(() -> metaStateService.cleanupGlobalState(generation)); - if (previousManifest.isGlobalGenerationMissing() == false) { - rollbackCleanupActions.add(() -> metaStateService.cleanupGlobalState(previousManifest.getGlobalGeneration())); - } + rollbackCleanupActions.add(() -> metaStateService.cleanupGlobalState(previousManifest.getGlobalGeneration())); return generation; } catch (WriteStateException e) { rollback(); @@ -226,10 +224,8 @@ long writeIndex(String reason, IndexMetaData metaData) throws WriteStateExceptio Index index = metaData.getIndex(); long generation = metaStateService.writeIndex(reason, metaData); commitCleanupActions.add(() -> metaStateService.cleanupIndex(index, generation)); - Long previousGeneration = previousManifest.getIndexGenerations().get(index); - if (previousGeneration != null) { - rollbackCleanupActions.add(() -> metaStateService.cleanupIndex(index, previousGeneration)); - } + long previousGeneration = previousManifest.getIndexGenerations().getOrDefault(index, -1L); + rollbackCleanupActions.add(() -> metaStateService.cleanupIndex(index, previousGeneration)); return generation; } catch (WriteStateException e) { rollback(); From a6dddc85ad8a5f6765c3922b8ff2a0109e19c83c Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Tue, 13 Nov 2018 09:48:02 +0300 Subject: [PATCH 43/57] Allow subclasses to override formats used in MetaStateService --- .../gateway/MetaStateService.java | 33 +++++++++++-------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java index 8397b0c0536d1..1d38155062dde 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java @@ -47,6 +47,11 @@ public class MetaStateService { private final NodeEnvironment nodeEnv; private final NamedXContentRegistry namedXContentRegistry; + // we allow subclasses in tests to redefine formats, e.g. to inject failures + protected MetaDataStateFormat META_DATA_FORMAT = MetaData.FORMAT; + protected MetaDataStateFormat INDEX_META_DATA_FORMAT = IndexMetaData.FORMAT; + protected MetaDataStateFormat MANIFEST_FORMAT = Manifest.FORMAT; + public MetaStateService(NodeEnvironment nodeEnv, NamedXContentRegistry namedXContentRegistry) { this.nodeEnv = nodeEnv; this.namedXContentRegistry = namedXContentRegistry; @@ -73,7 +78,7 @@ Tuple loadFullState() throws IOException { if (manifest.isGlobalGenerationMissing()) { metaDataBuilder = MetaData.builder(); } else { - final MetaData globalMetaData = MetaData.FORMAT.loadGeneration(logger, namedXContentRegistry, manifest.getGlobalGeneration(), + final MetaData globalMetaData = META_DATA_FORMAT.loadGeneration(logger, namedXContentRegistry, manifest.getGlobalGeneration(), nodeEnv.nodeDataPaths()); if (globalMetaData != null) { metaDataBuilder = MetaData.builder(globalMetaData); @@ -86,7 +91,7 @@ Tuple loadFullState() throws IOException { Index index = entry.getKey(); long generation = entry.getValue(); final String indexFolderName = index.getUUID(); - final IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadGeneration(logger, namedXContentRegistry, generation, + final IndexMetaData indexMetaData = INDEX_META_DATA_FORMAT.loadGeneration(logger, namedXContentRegistry, generation, nodeEnv.resolveIndexFolder(indexFolderName)); if (indexMetaData != null) { metaDataBuilder.put(indexMetaData, false); @@ -107,7 +112,7 @@ private Tuple loadFullStateBWC() throws IOException { MetaData.Builder metaDataBuilder; Tuple metaDataAndGeneration = - MetaData.FORMAT.loadLatestStateWithGeneration(logger, namedXContentRegistry, nodeEnv.nodeDataPaths()); + META_DATA_FORMAT.loadLatestStateWithGeneration(logger, namedXContentRegistry, nodeEnv.nodeDataPaths()); MetaData globalMetaData = metaDataAndGeneration.v1(); long globalStateGeneration = metaDataAndGeneration.v2(); @@ -120,7 +125,7 @@ private Tuple loadFullStateBWC() throws IOException { for (String indexFolderName : nodeEnv.availableIndexFolders()) { Tuple indexMetaDataAndGeneration = - IndexMetaData.FORMAT.loadLatestStateWithGeneration(logger, namedXContentRegistry, + INDEX_META_DATA_FORMAT.loadLatestStateWithGeneration(logger, namedXContentRegistry, nodeEnv.resolveIndexFolder(indexFolderName)); assert Version.CURRENT.major < 8 : "failed to find manifest file, which is mandatory staring with Elasticsearch version 8.0"; IndexMetaData indexMetaData = indexMetaDataAndGeneration.v1(); @@ -142,7 +147,7 @@ private Tuple loadFullStateBWC() throws IOException { */ @Nullable public IndexMetaData loadIndexState(Index index) throws IOException { - return IndexMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.indexPaths(index)); + return INDEX_META_DATA_FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.indexPaths(index)); } /** @@ -153,7 +158,7 @@ List loadIndicesStates(Predicate excludeIndexPathIdsPredi for (String indexFolderName : nodeEnv.availableIndexFolders(excludeIndexPathIdsPredicate)) { assert excludeIndexPathIdsPredicate.test(indexFolderName) == false : "unexpected folder " + indexFolderName + " which should have been excluded"; - IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, + IndexMetaData indexMetaData = INDEX_META_DATA_FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.resolveIndexFolder(indexFolderName)); if (indexMetaData != null) { final String indexPathId = indexMetaData.getIndex().getUUID(); @@ -173,7 +178,7 @@ List loadIndicesStates(Predicate excludeIndexPathIdsPredi * Loads Manifest file from disk, returns Manifest.empty() if there is no manifest file. */ public Manifest loadManifestOrEmpty() throws IOException { - Manifest manifest = Manifest.FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.nodeDataPaths()); + Manifest manifest = MANIFEST_FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.nodeDataPaths()); if (manifest == null) { manifest = Manifest.empty(); } @@ -184,7 +189,7 @@ public Manifest loadManifestOrEmpty() throws IOException { * Loads the global state, *without* index state, see {@link #loadFullState()} for that. */ MetaData loadGlobalState() throws IOException { - return MetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.nodeDataPaths()); + return META_DATA_FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.nodeDataPaths()); } /** @@ -195,7 +200,7 @@ MetaData loadGlobalState() throws IOException { public long writeManifest(String reason, Manifest manifest) throws WriteStateException { logger.trace("[_meta] writing state, reason [{}]", reason); try { - long generation = Manifest.FORMAT.write(manifest, nodeEnv.nodeDataPaths()); + long generation = MANIFEST_FORMAT.write(manifest, nodeEnv.nodeDataPaths()); logger.trace("[_meta] state written (generation: {})", generation); return generation; } catch (WriteStateException ex) { @@ -215,7 +220,7 @@ public long writeIndex(String reason, IndexMetaData indexMetaData) throws WriteS final Index index = indexMetaData.getIndex(); logger.trace("[{}] writing state, reason [{}]", index, reason); try { - long generation = IndexMetaData.FORMAT.write(indexMetaData, + long generation = INDEX_META_DATA_FORMAT.write(indexMetaData, nodeEnv.indexPaths(indexMetaData.getIndex())); logger.trace("[{}] state written", index); return generation; @@ -233,7 +238,7 @@ public long writeIndex(String reason, IndexMetaData indexMetaData) throws WriteS long writeGlobalState(String reason, MetaData metaData) throws WriteStateException { logger.trace("[_global] writing state, reason [{}]", reason); try { - long generation = MetaData.FORMAT.write(metaData, nodeEnv.nodeDataPaths()); + long generation = META_DATA_FORMAT.write(metaData, nodeEnv.nodeDataPaths()); logger.trace("[_global] state written"); return generation; } catch (WriteStateException ex) { @@ -247,7 +252,7 @@ long writeGlobalState(String reason, MetaData metaData) throws WriteStateExcepti * @param currentGeneration current state generation to keep in the directory. */ void cleanupGlobalState(long currentGeneration) { - MetaData.FORMAT.cleanupOldFiles(currentGeneration, nodeEnv.nodeDataPaths()); + META_DATA_FORMAT.cleanupOldFiles(currentGeneration, nodeEnv.nodeDataPaths()); } /** @@ -257,7 +262,7 @@ void cleanupGlobalState(long currentGeneration) { * @param currentGeneration current state generation to keep in the index directory. */ public void cleanupIndex(Index index, long currentGeneration) { - IndexMetaData.FORMAT.cleanupOldFiles(currentGeneration, nodeEnv.indexPaths(index)); + INDEX_META_DATA_FORMAT.cleanupOldFiles(currentGeneration, nodeEnv.indexPaths(index)); } /** @@ -266,7 +271,7 @@ public void cleanupIndex(Index index, long currentGeneration) { * @param currentGeneration current state generation to keep in the directory. */ public void cleanupManifest(long currentGeneration) { - Manifest.FORMAT.cleanupOldFiles(currentGeneration, nodeEnv.nodeDataPaths()); + MANIFEST_FORMAT.cleanupOldFiles(currentGeneration, nodeEnv.nodeDataPaths()); } /** From a3865f6196c1e94ba112c2e8f75251cd24486bfe Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Tue, 13 Nov 2018 09:48:49 +0300 Subject: [PATCH 44/57] Do not use loadManifestOrEmpty in loadFullState If used, even if empty Manifest file was written on the previous startup, loadFullState will fallback to BWC mode --- .../main/java/org/elasticsearch/gateway/MetaStateService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java index 1d38155062dde..4e10e9132a30b 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java @@ -69,8 +69,8 @@ public MetaStateService(NodeEnvironment nodeEnv, NamedXContentRegistry namedXCon * @throws IOException if some IOException when loading files occurs or there is no metadata referenced by manifest file. */ Tuple loadFullState() throws IOException { - final Manifest manifest = loadManifestOrEmpty(); - if (manifest.isEmpty()) { + final Manifest manifest = MANIFEST_FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.nodeDataPaths()); + if (manifest == null) { return loadFullStateBWC(); } From 7936cc4d43a8e288db613c1bd23c07c3d657ec93 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Tue, 13 Nov 2018 09:49:27 +0300 Subject: [PATCH 45/57] Add random atomicity test for Transaction --- .../gateway/MetaDataStateFormat.java | 4 + .../gateway/GatewayMetaStateTests.java | 152 +++++++++++++++++- 2 files changed, 153 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java b/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java index a3eba3d99ea5c..c610f97690796 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java @@ -456,4 +456,8 @@ public static void deleteMetaState(Path... dataLocations) throws IOException { } IOUtils.rm(stateDirectories); } + + public String getPrefix() { + return prefix; + } } diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java index 937849d14e04d..57df5a90944f8 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java @@ -19,11 +19,14 @@ package org.elasticsearch.gateway; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.MockDirectoryWrapper; import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ESAllocationTestCase; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; +import org.elasticsearch.cluster.metadata.Manifest; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -31,13 +34,19 @@ import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.Index; import org.elasticsearch.plugins.MetaDataUpgrader; import org.elasticsearch.test.TestCustomMetaData; import org.mockito.ArgumentCaptor; -import java.util.ArrayList; +import java.io.IOException; +import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; @@ -246,8 +255,6 @@ public void testResolveStatesToBeWritten() throws WriteStateException { List actions = GatewayMetaState.resolveIndexMetaDataActions(indices, relevantIndices, oldMetaData, newMetaData); - List cleanupActions = new ArrayList<>(); - assertThat(actions, hasSize(3)); for (GatewayMetaState.IndexMetaDataAction action : actions) { @@ -276,6 +283,145 @@ public void testResolveStatesToBeWritten() throws WriteStateException { } } + private static class MetaStateServiceWithFailures extends MetaStateService { + private final int invertedFailRate; + private boolean failRandomly; + + private MetaDataStateFormat wrap(MetaDataStateFormat format) { + return new MetaDataStateFormat(format.getPrefix()) { + @Override + public void toXContent(XContentBuilder builder, T state) throws IOException { + format.toXContent(builder, state); + } + + @Override + public T fromXContent(XContentParser parser) throws IOException { + return format.fromXContent(parser); + } + + @Override + protected Directory newDirectory(Path dir) { + MockDirectoryWrapper mock = newMockFSDirectory(dir); + if (failRandomly) { + MockDirectoryWrapper.Failure fail = new MockDirectoryWrapper.Failure() { + @Override + public void eval(MockDirectoryWrapper dir) throws IOException { + int r = randomIntBetween(0, invertedFailRate); + if (r == 0) { + throw new MockDirectoryWrapper.FakeIOException(); + } + } + }; + mock.failOn(fail); + } + closeAfterSuite(mock); + return mock; + } + }; + } + + MetaStateServiceWithFailures(int invertedFailRate, NodeEnvironment nodeEnv, NamedXContentRegistry namedXContentRegistry) { + super(nodeEnv, namedXContentRegistry); + META_DATA_FORMAT = wrap(MetaData.FORMAT); + INDEX_META_DATA_FORMAT = wrap(IndexMetaData.FORMAT); + MANIFEST_FORMAT = wrap(Manifest.FORMAT); + failRandomly = false; + this.invertedFailRate = invertedFailRate; + } + + void failRandomly() { + failRandomly = true; + } + + void noFailures() { + failRandomly = false; + } + } + + private boolean metaDataEquals(MetaData md1, MetaData md2) { + boolean equals = MetaData.isGlobalStateEquals(md1, md2); + + for (IndexMetaData imd : md1) { + IndexMetaData imd2 = md2.index(imd.getIndex()); + equals = equals && imd.equals(imd2); + } + + for (IndexMetaData imd : md2) { + IndexMetaData imd2 = md1.index(imd.getIndex()); + equals = equals && imd.equals(imd2); + } + return equals; + } + + private MetaData randomMetaData() { + int settingNo = randomIntBetween(0, 10); + MetaData.Builder builder = MetaData.builder() + .persistentSettings(Settings.builder().put("setting" + settingNo, randomAlphaOfLength(5)).build()); + int numOfIndices = randomIntBetween(0, 3); + + for (int i = 0; i < numOfIndices; i++) { + int indexNo = randomIntBetween(0, 50); + IndexMetaData indexMetaData = IndexMetaData.builder("index" + indexNo).settings( + Settings.builder() + .put(IndexMetaData.SETTING_INDEX_UUID, "index" + indexNo) + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .build() + ).build(); + builder.put(indexMetaData, false); + } + return builder.build(); + } + + public void testTransactionAtomicityWithFailures() throws IOException { + try (NodeEnvironment env = newNodeEnvironment()) { + MetaStateServiceWithFailures metaStateService = + new MetaStateServiceWithFailures(randomIntBetween(100, 1000), env, xContentRegistry()); + + // We only guarantee atomicity of writes, if there is initial Manifest file + Manifest manifest = Manifest.empty(); + MetaData metaData = MetaData.EMPTY_META_DATA; + metaStateService.writeManifest("startup", Manifest.empty()); + + metaStateService.failRandomly(); + Set possibleMetaData = new HashSet<>(); + possibleMetaData.add(metaData); + + for (int i = 0; i < randomIntBetween(1, 5); i++) { + GatewayMetaState.Transaction tx = new GatewayMetaState.Transaction(metaStateService, manifest); + metaData = randomMetaData(); + Map indexGenerations = new HashMap<>(); + + try { + long globalGeneration = tx.writeGlobalState("global", metaData); + + for (IndexMetaData indexMetaData : metaData) { + long generation = tx.writeIndex("index", indexMetaData); + indexGenerations.put(indexMetaData.getIndex(), generation); + } + + Manifest newManifest = new Manifest(globalGeneration, indexGenerations); + tx.writeManifest("manifest", newManifest); + possibleMetaData.clear(); + possibleMetaData.add(metaData); + manifest = newManifest; + } catch (WriteStateException e) { + if (e.isDirty()) { + possibleMetaData.add(metaData); + } + } + } + + metaStateService.noFailures(); + + Tuple manifestAndMetaData = metaStateService.loadFullState(); + MetaData loadedMetaData = manifestAndMetaData.v2(); + + assertTrue(possibleMetaData.stream().anyMatch(md -> metaDataEquals(md, loadedMetaData))); + } + } + public void testAddCustomMetaDataOnUpgrade() throws Exception { MetaData metaData = randomMetaData(); MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader( From ae1951ecb5c17d408fc955c6c087c5f1c9311f2d Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Tue, 13 Nov 2018 15:15:37 +0300 Subject: [PATCH 46/57] randomMetaData -> randomMetaDataForTx, where is already randomMetaData --- .../java/org/elasticsearch/gateway/GatewayMetaStateTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java index 57df5a90944f8..383075bb4a861 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java @@ -353,7 +353,7 @@ private boolean metaDataEquals(MetaData md1, MetaData md2) { return equals; } - private MetaData randomMetaData() { + private static MetaData randomMetaDataForTx() { int settingNo = randomIntBetween(0, 10); MetaData.Builder builder = MetaData.builder() .persistentSettings(Settings.builder().put("setting" + settingNo, randomAlphaOfLength(5)).build()); @@ -390,7 +390,7 @@ public void testTransactionAtomicityWithFailures() throws IOException { for (int i = 0; i < randomIntBetween(1, 5); i++) { GatewayMetaState.Transaction tx = new GatewayMetaState.Transaction(metaStateService, manifest); - metaData = randomMetaData(); + metaData = randomMetaDataForTx(); Map indexGenerations = new HashMap<>(); try { From 326be60539111cb922c973727dd890ea09747f48 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Tue, 13 Nov 2018 19:11:51 +0300 Subject: [PATCH 47/57] write should not perform cleanup at all, writeAndCleanup should always --- .../gateway/MetaDataStateFormat.java | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java b/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java index c610f97690796..1326274e82ea2 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java @@ -177,13 +177,12 @@ private static void performStateDirectoriesFsync(List> st } /** - * Writes the given state to the given directories and performs cleanup. + * Writes the given state to the given directories and performs cleanup of old state files if the write succeeds or + * newly created state file if write fails. * See also {@link #write(Object, Path...)} and {@link #cleanupOldFiles(long, Path[])}. */ public final long writeAndCleanup(final T state, final Path... locations) throws WriteStateException { - long currentGeneration = write(state, locations); - cleanupOldFiles(currentGeneration,locations); - return currentGeneration; + return write(state, true, locations); } /** @@ -207,6 +206,10 @@ public final long writeAndCleanup(final T state, final Path... locations) throws * @return generation of newly written state. */ public final long write(final T state, final Path... locations) throws WriteStateException { + return write(state, false, locations); + } + + private final long write(final T state, boolean cleanup, final Path... locations) throws WriteStateException { if (locations == null) { throw new IllegalArgumentException("Locations must not be null"); } @@ -242,7 +245,9 @@ public final long write(final T state, final Path... locations) throws WriteStat performRenames(tmpFileName, fileName, directories); performStateDirectoriesFsync(directories); } catch (WriteStateException e) { - cleanupOldFiles(oldGenerationId, locations); + if (cleanup) { + cleanupOldFiles(oldGenerationId, locations); + } throw e; } finally { for (Tuple pathAndDirectory : directories) { @@ -251,6 +256,10 @@ public final long write(final T state, final Path... locations) throws WriteStat } } + if (cleanup) { + cleanupOldFiles(newGenerationId, locations); + } + return newGenerationId; } From 982e853dfef534e07a35c56478c20019c2161d01 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Tue, 13 Nov 2018 19:12:59 +0300 Subject: [PATCH 48/57] Use writeManifestAndCleanup, reorder rollbackActions before write --- .../gateway/GatewayMetaState.java | 17 +++++++------- .../gateway/MetaStateService.java | 22 +++++-------------- .../gateway/GatewayMetaStateTests.java | 4 ++-- .../gateway/MetaStateServiceTests.java | 9 ++++---- 4 files changed, 20 insertions(+), 32 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index db5d54f5fae51..162c91926eda2 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -135,7 +135,7 @@ private void upgradeMetaData(MetaDataIndexUpgradeService metaDataIndexUpgradeSer } final Manifest newManifest = new Manifest(globalStateGeneration, indices); - tx.writeManifest("startup", newManifest); + tx.writeManifestAndCleanup("startup", newManifest); } catch (Exception e) { logger.error("failed to read or re-write local state, exiting...", e); throw e; @@ -184,7 +184,7 @@ public void applyClusterState(ClusterChangedEvent event) { * This class is used to write changed global {@link MetaData}, {@link IndexMetaData} and {@link Manifest} to disk. * This class delegates write* calls to corresponding write calls in {@link MetaStateService} and * additionally it keeps track of cleanup actions to be performed if transaction succeeds or fails. - * Transaction succeeds if {@link #writeManifest(String, Manifest)} call succeeds, transaction fails if + * Transaction succeeds if {@link #writeManifestAndCleanup(String, Manifest)} call succeeds, transaction fails if * any write* call fails. * Once transaction succeeds/fails it can no longer be used. */ @@ -208,9 +208,9 @@ static class Transaction { long writeGlobalState(String reason, MetaData metaData) throws WriteStateException { assert finished == false : TRANSACTION_COMPLETED_MSG; try { + rollbackCleanupActions.add(() -> metaStateService.cleanupGlobalState(previousManifest.getGlobalGeneration())); long generation = metaStateService.writeGlobalState(reason, metaData); commitCleanupActions.add(() -> metaStateService.cleanupGlobalState(generation)); - rollbackCleanupActions.add(() -> metaStateService.cleanupGlobalState(previousManifest.getGlobalGeneration())); return generation; } catch (WriteStateException e) { rollback(); @@ -222,10 +222,10 @@ long writeIndex(String reason, IndexMetaData metaData) throws WriteStateExceptio assert finished == false : TRANSACTION_COMPLETED_MSG; try { Index index = metaData.getIndex(); - long generation = metaStateService.writeIndex(reason, metaData); - commitCleanupActions.add(() -> metaStateService.cleanupIndex(index, generation)); long previousGeneration = previousManifest.getIndexGenerations().getOrDefault(index, -1L); rollbackCleanupActions.add(() -> metaStateService.cleanupIndex(index, previousGeneration)); + long generation = metaStateService.writeIndex(reason, metaData); + commitCleanupActions.add(() -> metaStateService.cleanupIndex(index, generation)); return generation; } catch (WriteStateException e) { rollback(); @@ -233,11 +233,10 @@ long writeIndex(String reason, IndexMetaData metaData) throws WriteStateExceptio } } - long writeManifest(String reason, Manifest manifest) throws WriteStateException { + long writeManifestAndCleanup(String reason, Manifest manifest) throws WriteStateException { assert finished == false : TRANSACTION_COMPLETED_MSG; try { - long generation = metaStateService.writeManifest(reason, manifest); - commitCleanupActions.add(() -> metaStateService.cleanupManifest(generation)); + long generation = metaStateService.writeManifestAndCleanup(reason, manifest); commitCleanupActions.forEach(Runnable::run); finished = true; return generation; @@ -276,7 +275,7 @@ private void updateMetaData(ClusterChangedEvent event) throws IOException { private void writeManifest(Transaction tx, Manifest manifest) throws IOException { if (manifest.equals(previousManifest) == false) { - tx.writeManifest("changed", manifest); + tx.writeManifestAndCleanup("changed", manifest); } } diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java index 4e10e9132a30b..40f18af0cf01b 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java @@ -193,14 +193,15 @@ MetaData loadGlobalState() throws IOException { } /** - * Writes manifest file (represented by {@link Manifest}) to disk. + * Writes manifest file (represented by {@link Manifest}) to disk and performs cleanup of old manifest state file if + * the write succeeds or newly created manifest state if the write fails. * * @throws WriteStateException if exception when writing state occurs. See also {@link WriteStateException#isDirty()} */ - public long writeManifest(String reason, Manifest manifest) throws WriteStateException { + public long writeManifestAndCleanup(String reason, Manifest manifest) throws WriteStateException { logger.trace("[_meta] writing state, reason [{}]", reason); try { - long generation = MANIFEST_FORMAT.write(manifest, nodeEnv.nodeDataPaths()); + long generation = MANIFEST_FORMAT.writeAndCleanup(manifest, nodeEnv.nodeDataPaths()); logger.trace("[_meta] state written (generation: {})", generation); return generation; } catch (WriteStateException ex) { @@ -265,15 +266,6 @@ public void cleanupIndex(Index index, long currentGeneration) { INDEX_META_DATA_FORMAT.cleanupOldFiles(currentGeneration, nodeEnv.indexPaths(index)); } - /** - * Removes old state files in meta state directory. - * - * @param currentGeneration current state generation to keep in the directory. - */ - public void cleanupManifest(long currentGeneration) { - MANIFEST_FORMAT.cleanupOldFiles(currentGeneration, nodeEnv.nodeDataPaths()); - } - /** * Writes index metadata and updates manifest file accordingly. * Used by tests. @@ -284,9 +276,8 @@ public void writeIndexAndUpdateManifest(String reason, IndexMetaData metaData) t Map indices = new HashMap<>(manifest.getIndexGenerations()); indices.put(metaData.getIndex(), generation); manifest = new Manifest(manifest.getGlobalGeneration(), indices); - long metaStateGeneration = writeManifest(reason, manifest); + writeManifestAndCleanup(reason, manifest); cleanupIndex(metaData.getIndex(), generation); - cleanupManifest(metaStateGeneration); } /** @@ -297,8 +288,7 @@ public void writeGlobalStateAndUpdateManifest(String reason, MetaData metaData) long generation = writeGlobalState(reason, metaData); Manifest manifest = loadManifestOrEmpty(); manifest = new Manifest(generation, manifest.getIndexGenerations()); - long metaStateGeneration = writeManifest(reason, manifest); + writeManifestAndCleanup(reason, manifest); cleanupGlobalState(generation); - cleanupManifest(metaStateGeneration); } } \ No newline at end of file diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java index 383075bb4a861..9157a665b063d 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java @@ -382,7 +382,7 @@ public void testTransactionAtomicityWithFailures() throws IOException { // We only guarantee atomicity of writes, if there is initial Manifest file Manifest manifest = Manifest.empty(); MetaData metaData = MetaData.EMPTY_META_DATA; - metaStateService.writeManifest("startup", Manifest.empty()); + metaStateService.writeManifestAndCleanup("startup", Manifest.empty()); metaStateService.failRandomly(); Set possibleMetaData = new HashSet<>(); @@ -402,7 +402,7 @@ public void testTransactionAtomicityWithFailures() throws IOException { } Manifest newManifest = new Manifest(globalGeneration, indexGenerations); - tx.writeManifest("manifest", newManifest); + tx.writeManifestAndCleanup("manifest", newManifest); possibleMetaData.clear(); possibleMetaData.add(metaData); manifest = newManifest; diff --git a/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java b/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java index 1081a311f6084..28dd2f8ba1759 100644 --- a/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java @@ -129,7 +129,7 @@ public void testLoadEmptyStateNoManifest() throws IOException { public void testLoadEmptyStateWithManifest() throws IOException { Manifest manifest = Manifest.empty(); - metaStateService.writeManifest("test", manifest); + metaStateService.writeManifestAndCleanup("test", manifest); Tuple manifestAndMetaData = metaStateService.loadFullState(); assertTrue(manifestAndMetaData.v1().isEmpty()); @@ -144,7 +144,7 @@ public void testLoadFullStateMissingGlobalMetaData() throws IOException { put(index.getIndex(), indexGeneration); }}); assertTrue(manifest.isGlobalGenerationMissing()); - metaStateService.writeManifest("test", manifest); + metaStateService.writeManifestAndCleanup("test", manifest); Tuple manifestAndMetaData = metaStateService.loadFullState(); assertThat(manifestAndMetaData.v1(), equalTo(manifest)); @@ -168,7 +168,7 @@ public void testLoadFullStateAndUpdate() throws IOException { put(index.getIndex(), indexGeneration); }}); - metaStateService.writeManifest("first manifest write", manifest); + metaStateService.writeManifestAndCleanup("first manifest write", manifest); MetaData newMetaData = MetaData.builder() .persistentSettings(Settings.builder().put("test1", "value2").build()) @@ -188,10 +188,9 @@ public void testLoadFullStateAndUpdate() throws IOException { put(index.getIndex(), indexGeneration); }}); - long manifestGeneration = metaStateService.writeManifest("second manifest write", manifest); + metaStateService.writeManifestAndCleanup("second manifest write", manifest); metaStateService.cleanupGlobalState(globalGeneration); metaStateService.cleanupIndex(index.getIndex(), indexGeneration); - metaStateService.cleanupManifest(manifestGeneration); manifestAndMetaData = metaStateService.loadFullState(); assertThat(manifestAndMetaData.v1(), equalTo(manifest)); From 4113830e20383adcb7ac70747bcbeaa14544ef61 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Tue, 13 Nov 2018 19:35:14 +0300 Subject: [PATCH 49/57] Transaction -> AtomicClusterStateWriter --- .../gateway/GatewayMetaState.java | 57 +++++++++---------- .../gateway/GatewayMetaStateTests.java | 28 ++++----- 2 files changed, 41 insertions(+), 44 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index 162c91926eda2..8683bf3c58033 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -116,12 +116,12 @@ private void upgradeMetaData(MetaDataIndexUpgradeService metaDataIndexUpgradeSer // if there is manifest file, it means metadata is properly persisted to all data paths // if there is no manifest file (upgrade from 6.x to 7.x) metadata might be missing on some data paths, // but anyway we will re-write it as soon as we receive first ClusterState - final Transaction tx = new Transaction(metaStateService, manifest); + final AtomicClusterStateWriter writer = new AtomicClusterStateWriter(metaStateService, manifest); final MetaData upgradedMetaData = upgradeMetaData(metaData, metaDataIndexUpgradeService, metaDataUpgrader); final long globalStateGeneration; if (MetaData.isGlobalStateEquals(metaData, upgradedMetaData) == false) { - globalStateGeneration = tx.writeGlobalState("upgrade", upgradedMetaData); + globalStateGeneration = writer.writeGlobalState("upgrade", upgradedMetaData); } else { globalStateGeneration = manifest.getGlobalGeneration(); } @@ -129,13 +129,13 @@ private void upgradeMetaData(MetaDataIndexUpgradeService metaDataIndexUpgradeSer Map indices = new HashMap<>(manifest.getIndexGenerations()); for (IndexMetaData indexMetaData : upgradedMetaData) { if (metaData.hasIndexMetaData(indexMetaData) == false) { - final long generation = tx.writeIndex("upgrade", indexMetaData); + final long generation = writer.writeIndex("upgrade", indexMetaData); indices.put(indexMetaData.getIndex(), generation); } } final Manifest newManifest = new Manifest(globalStateGeneration, indices); - tx.writeManifestAndCleanup("startup", newManifest); + writer.writeManifestAndCleanup("startup", newManifest); } catch (Exception e) { logger.error("failed to read or re-write local state, exiting...", e); throw e; @@ -184,19 +184,16 @@ public void applyClusterState(ClusterChangedEvent event) { * This class is used to write changed global {@link MetaData}, {@link IndexMetaData} and {@link Manifest} to disk. * This class delegates write* calls to corresponding write calls in {@link MetaStateService} and * additionally it keeps track of cleanup actions to be performed if transaction succeeds or fails. - * Transaction succeeds if {@link #writeManifestAndCleanup(String, Manifest)} call succeeds, transaction fails if - * any write* call fails. - * Once transaction succeeds/fails it can no longer be used. */ - static class Transaction { - private static final String TRANSACTION_COMPLETED_MSG = "Transaction is already completed"; + static class AtomicClusterStateWriter { + private static final String FINISHED_MSG = "AtomicClusterStateWriter is finished"; private final List commitCleanupActions; private final List rollbackCleanupActions; private final Manifest previousManifest; private final MetaStateService metaStateService; private boolean finished; - Transaction(MetaStateService metaStateService, Manifest previousManifest) { + AtomicClusterStateWriter(MetaStateService metaStateService, Manifest previousManifest) { this.metaStateService = metaStateService; assert previousManifest != null; this.previousManifest = previousManifest; @@ -206,7 +203,7 @@ static class Transaction { } long writeGlobalState(String reason, MetaData metaData) throws WriteStateException { - assert finished == false : TRANSACTION_COMPLETED_MSG; + assert finished == false : FINISHED_MSG; try { rollbackCleanupActions.add(() -> metaStateService.cleanupGlobalState(previousManifest.getGlobalGeneration())); long generation = metaStateService.writeGlobalState(reason, metaData); @@ -219,7 +216,7 @@ long writeGlobalState(String reason, MetaData metaData) throws WriteStateExcepti } long writeIndex(String reason, IndexMetaData metaData) throws WriteStateException { - assert finished == false : TRANSACTION_COMPLETED_MSG; + assert finished == false : FINISHED_MSG; try { Index index = metaData.getIndex(); long previousGeneration = previousManifest.getIndexGenerations().getOrDefault(index, -1L); @@ -234,7 +231,7 @@ long writeIndex(String reason, IndexMetaData metaData) throws WriteStateExceptio } long writeManifestAndCleanup(String reason, Manifest manifest) throws WriteStateException { - assert finished == false : TRANSACTION_COMPLETED_MSG; + assert finished == false : FINISHED_MSG; try { long generation = metaStateService.writeManifestAndCleanup(reason, manifest); commitCleanupActions.forEach(Runnable::run); @@ -263,23 +260,23 @@ private void updateMetaData(ClusterChangedEvent event) throws IOException { ClusterState previousState = event.previousState(); MetaData newMetaData = newState.metaData(); - final Transaction tx = new Transaction(metaStateService, previousManifest); - long globalStateGeneration = writeGlobalState(tx, newMetaData); - Map indexGenerations = writeIndicesMetadata(tx, newState, previousState); + final AtomicClusterStateWriter writer = new AtomicClusterStateWriter(metaStateService, previousManifest); + long globalStateGeneration = writeGlobalState(writer, newMetaData); + Map indexGenerations = writeIndicesMetadata(writer, newState, previousState); Manifest manifest = new Manifest(globalStateGeneration, indexGenerations); - writeManifest(tx, manifest); + writeManifest(writer, manifest); previousMetaData = newMetaData; previousManifest = manifest; } - private void writeManifest(Transaction tx, Manifest manifest) throws IOException { + private void writeManifest(AtomicClusterStateWriter writer, Manifest manifest) throws IOException { if (manifest.equals(previousManifest) == false) { - tx.writeManifestAndCleanup("changed", manifest); + writer.writeManifestAndCleanup("changed", manifest); } } - private Map writeIndicesMetadata(Transaction tx, ClusterState newState, ClusterState previousState) + private Map writeIndicesMetadata(AtomicClusterStateWriter writer, ClusterState newState, ClusterState previousState) throws IOException { Map previouslyWrittenIndices = previousManifest.getIndexGenerations(); Set relevantIndices = getRelevantIndices(newState, previousState, previouslyWrittenIndices.keySet()); @@ -290,16 +287,16 @@ private Map writeIndicesMetadata(Transaction tx, ClusterState newSt newState.metaData()); for (IndexMetaDataAction action : actions) { - long generation = action.execute(tx); + long generation = action.execute(writer); newIndices.put(action.getIndex(), generation); } return newIndices; } - private long writeGlobalState(Transaction tx, MetaData newMetaData) throws IOException { + private long writeGlobalState(AtomicClusterStateWriter writer, MetaData newMetaData) throws IOException { if (previousMetaData == null || MetaData.isGlobalStateEquals(previousMetaData, newMetaData) == false) { - return tx.writeGlobalState("changed", newMetaData); + return writer.writeGlobalState("changed", newMetaData); } return previousManifest.getGlobalGeneration(); } @@ -517,12 +514,12 @@ public interface IndexMetaDataAction { Index getIndex(); /** - * Executes this action using provided {@link Transaction}. + * Executes this action using provided {@link AtomicClusterStateWriter}. * * @return new index metadata state generation, to be used in manifest file. * @throws WriteStateException if exception occurs. */ - long execute(Transaction tx) throws WriteStateException; + long execute(AtomicClusterStateWriter writer) throws WriteStateException; } public static class KeepPreviousGeneration implements IndexMetaDataAction { @@ -540,7 +537,7 @@ public Index getIndex() { } @Override - public long execute(Transaction tx) { + public long execute(AtomicClusterStateWriter writer) { return generation; } } @@ -558,8 +555,8 @@ public Index getIndex() { } @Override - public long execute(Transaction tx) throws WriteStateException { - return tx.writeIndex("freshly created", indexMetaData); + public long execute(AtomicClusterStateWriter writer) throws WriteStateException { + return writer.writeIndex("freshly created", indexMetaData); } } @@ -578,8 +575,8 @@ public Index getIndex() { } @Override - public long execute(Transaction tx) throws WriteStateException { - return tx.writeIndex( + public long execute(AtomicClusterStateWriter writer) throws WriteStateException { + return writer.writeIndex( "version changed from [" + oldIndexMetaData.getVersion() + "] to [" + newIndexMetaData.getVersion() + "]", newIndexMetaData); } diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java index 9157a665b063d..3699cc3e3fcd0 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java @@ -260,23 +260,23 @@ public void testResolveStatesToBeWritten() throws WriteStateException { for (GatewayMetaState.IndexMetaDataAction action : actions) { if (action instanceof GatewayMetaState.KeepPreviousGeneration) { assertThat(action.getIndex(), equalTo(notChangedIndex.getIndex())); - GatewayMetaState.Transaction tx = mock(GatewayMetaState.Transaction.class); - assertThat(action.execute(tx), equalTo(3L)); - verifyZeroInteractions(tx); + GatewayMetaState.AtomicClusterStateWriter writer = mock(GatewayMetaState.AtomicClusterStateWriter.class); + assertThat(action.execute(writer), equalTo(3L)); + verifyZeroInteractions(writer); } if (action instanceof GatewayMetaState.WriteNewIndexMetaData) { assertThat(action.getIndex(), equalTo(newIndex.getIndex())); - GatewayMetaState.Transaction tx = mock(GatewayMetaState.Transaction.class); - when(tx.writeIndex("freshly created", newIndex)).thenReturn(0L); - assertThat(action.execute(tx), equalTo(0L)); + GatewayMetaState.AtomicClusterStateWriter writer = mock(GatewayMetaState.AtomicClusterStateWriter.class); + when(writer.writeIndex("freshly created", newIndex)).thenReturn(0L); + assertThat(action.execute(writer), equalTo(0L)); } if (action instanceof GatewayMetaState.WriteChangedIndexMetaData) { assertThat(action.getIndex(), equalTo(newVersionChangedIndex.getIndex())); - GatewayMetaState.Transaction tx = mock(GatewayMetaState.Transaction.class); - when(tx.writeIndex(anyString(), eq(newVersionChangedIndex))).thenReturn(3L); - assertThat(action.execute(tx), equalTo(3L)); + GatewayMetaState.AtomicClusterStateWriter writer = mock(GatewayMetaState.AtomicClusterStateWriter.class); + when(writer.writeIndex(anyString(), eq(newVersionChangedIndex))).thenReturn(3L); + assertThat(action.execute(writer), equalTo(3L)); ArgumentCaptor reason = ArgumentCaptor.forClass(String.class); - verify(tx).writeIndex(reason.capture(), eq(newVersionChangedIndex)); + verify(writer).writeIndex(reason.capture(), eq(newVersionChangedIndex)); assertThat(reason.getValue(), containsString(Long.toString(versionChangedIndex.getVersion()))); assertThat(reason.getValue(), containsString(Long.toString(newVersionChangedIndex.getVersion()))); } @@ -389,20 +389,20 @@ public void testTransactionAtomicityWithFailures() throws IOException { possibleMetaData.add(metaData); for (int i = 0; i < randomIntBetween(1, 5); i++) { - GatewayMetaState.Transaction tx = new GatewayMetaState.Transaction(metaStateService, manifest); + GatewayMetaState.AtomicClusterStateWriter writer = new GatewayMetaState.AtomicClusterStateWriter(metaStateService, manifest); metaData = randomMetaDataForTx(); Map indexGenerations = new HashMap<>(); try { - long globalGeneration = tx.writeGlobalState("global", metaData); + long globalGeneration = writer.writeGlobalState("global", metaData); for (IndexMetaData indexMetaData : metaData) { - long generation = tx.writeIndex("index", indexMetaData); + long generation = writer.writeIndex("index", indexMetaData); indexGenerations.put(indexMetaData.getIndex(), generation); } Manifest newManifest = new Manifest(globalGeneration, indexGenerations); - tx.writeManifestAndCleanup("manifest", newManifest); + writer.writeManifestAndCleanup("manifest", newManifest); possibleMetaData.clear(); possibleMetaData.add(metaData); manifest = newManifest; From 0e63bdcece72dbc13e8f166aa55e4c406bd92f0d Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Tue, 13 Nov 2018 19:47:00 +0300 Subject: [PATCH 50/57] Fix style --- .../java/org/elasticsearch/gateway/MetaDataStateFormat.java | 2 +- .../java/org/elasticsearch/gateway/GatewayMetaStateTests.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java b/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java index 1326274e82ea2..45bbaf66d8fda 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java @@ -209,7 +209,7 @@ public final long write(final T state, final Path... locations) throws WriteStat return write(state, false, locations); } - private final long write(final T state, boolean cleanup, final Path... locations) throws WriteStateException { + private long write(final T state, boolean cleanup, final Path... locations) throws WriteStateException { if (locations == null) { throw new IllegalArgumentException("Locations must not be null"); } diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java index 3699cc3e3fcd0..b9cf33f0d5156 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java @@ -389,7 +389,8 @@ public void testTransactionAtomicityWithFailures() throws IOException { possibleMetaData.add(metaData); for (int i = 0; i < randomIntBetween(1, 5); i++) { - GatewayMetaState.AtomicClusterStateWriter writer = new GatewayMetaState.AtomicClusterStateWriter(metaStateService, manifest); + GatewayMetaState.AtomicClusterStateWriter writer = + new GatewayMetaState.AtomicClusterStateWriter(metaStateService, manifest); metaData = randomMetaDataForTx(); Map indexGenerations = new HashMap<>(); From 0bc5ab55c7d6479839cafd43ed189d5bc1802306 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Tue, 13 Nov 2018 19:56:47 +0300 Subject: [PATCH 51/57] Do not cleanup non-referenced IMD in case of rollback --- .../java/org/elasticsearch/gateway/GatewayMetaState.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index 8683bf3c58033..a3466ba25e5ce 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -219,8 +219,13 @@ long writeIndex(String reason, IndexMetaData metaData) throws WriteStateExceptio assert finished == false : FINISHED_MSG; try { Index index = metaData.getIndex(); - long previousGeneration = previousManifest.getIndexGenerations().getOrDefault(index, -1L); - rollbackCleanupActions.add(() -> metaStateService.cleanupIndex(index, previousGeneration)); + Long previousGeneration = previousManifest.getIndexGenerations().get(index); + if (previousGeneration != null) { + // we prefer not to clean-up index metadata in case of rollback, + // if it's not referenced by previous manifest file + // not to break dandling indices functionality + rollbackCleanupActions.add(() -> metaStateService.cleanupIndex(index, previousGeneration)); + } long generation = metaStateService.writeIndex(reason, metaData); commitCleanupActions.add(() -> metaStateService.cleanupIndex(index, generation)); return generation; From 17bb22a0f677f5680ffe688d35fe7b055b4c2c82 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Wed, 14 Nov 2018 15:41:51 +0300 Subject: [PATCH 52/57] Revert "fullRestart should call onNodeStopped callback, before re-creating nodes" This reverts commit f21ddf9 --- .../test/InternalTestCluster.java | 38 +++++++------------ 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index f36833c0ab55e..1f71a2c2d9b6f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -923,9 +923,18 @@ void restart(RestartCallback callback, boolean clearDataIfNeeded, int minMasterN if (!node.isClosed()) { closeNode(); } + recreateNodeOnRestart(callback, clearDataIfNeeded, minMasterNodes, () -> rebuildUnicastHostFiles(emptyList())); + startNode(); + } + + /** + * rebuilds a new node object using the current node settings and starts it + */ + void recreateNodeOnRestart(RestartCallback callback, boolean clearDataIfNeeded, int minMasterNodes, + Runnable onTransportServiceStarted) throws Exception { assert callback != null; Settings callbackSettings = callback.onNodeStopped(name); - Builder newSettings = Settings.builder(); + Settings.Builder newSettings = Settings.builder(); if (callbackSettings != null) { newSettings.put(callbackSettings); } @@ -936,10 +945,9 @@ void restart(RestartCallback callback, boolean clearDataIfNeeded, int minMasterN if (clearDataIfNeeded) { clearDataIfNeeded(callback); } - createNewNode(newSettings.build(), () -> rebuildUnicastHostFiles(emptyList())); + createNewNode(newSettings.build(), onTransportServiceStarted); // make sure cached client points to new node resetClient(); - startNode(); } private void clearDataIfNeeded(RestartCallback callback) throws IOException { @@ -1757,29 +1765,11 @@ public synchronized void fullRestart(RestartCallback callback) throws Exception } assert nodesByRoles.values().stream().collect(Collectors.summingInt(List::size)) == 0; - //Let's call onNodeStopped callback on all nodes, before re-creating the nodes - final Map newNodeSettings = new HashMap<>(); - for (NodeAndClient nodeAndClient : startUpOrder) { - Settings callbackSettings = callback.onNodeStopped(nodeAndClient.name); - Builder newSettings = Settings.builder(); - if (callbackSettings != null) { - newSettings.put(callbackSettings); - } - if (autoManageMinMasterNodes) { - int minMasterNodes = getMinMasterNodes(getMasterNodesCount()); - assert DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(newSettings.build()) == false : "min master nodes is auto managed"; - newSettings.put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minMasterNodes).build(); - } - newNodeSettings.put(nodeAndClient.name, newSettings.build()); - } - - //Now we can re-create the nodes for (NodeAndClient nodeAndClient : startUpOrder) { logger.info("resetting node [{}] ", nodeAndClient.name); - - nodeAndClient.createNewNode(newNodeSettings.get(nodeAndClient.name), () -> rebuildUnicastHostFiles(startUpOrder)); - // make sure cached client points to new node - nodeAndClient.resetClient(); + // we already cleared data folders, before starting nodes up + nodeAndClient.recreateNodeOnRestart(callback, false, autoManageMinMasterNodes ? getMinMasterNodes(getMasterNodesCount()) : -1, + () -> rebuildUnicastHostFiles(startUpOrder)); } startAndPublishNodesAndClients(startUpOrder); From c7fb593e320228385a0800b807184740b3f8a6ac Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Thu, 15 Nov 2018 12:25:44 +0300 Subject: [PATCH 53/57] getPrefix package private --- .../java/org/elasticsearch/gateway/MetaDataStateFormat.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java b/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java index 45bbaf66d8fda..3f28fead29439 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java @@ -466,7 +466,7 @@ public static void deleteMetaState(Path... dataLocations) throws IOException { IOUtils.rm(stateDirectories); } - public String getPrefix() { + String getPrefix() { return prefix; } } From 736bde53911c2760b60d3f72a5987fc6abe57c7e Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Thu, 15 Nov 2018 12:26:52 +0300 Subject: [PATCH 54/57] Add final modifiers --- .../main/java/org/elasticsearch/gateway/MetaStateService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java index 40f18af0cf01b..7c4b1cbb33ab6 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java @@ -88,8 +88,8 @@ Tuple loadFullState() throws IOException { } for (Map.Entry entry : manifest.getIndexGenerations().entrySet()) { - Index index = entry.getKey(); - long generation = entry.getValue(); + final Index index = entry.getKey(); + final long generation = entry.getValue(); final String indexFolderName = index.getUUID(); final IndexMetaData indexMetaData = INDEX_META_DATA_FORMAT.loadGeneration(logger, namedXContentRegistry, generation, nodeEnv.resolveIndexFolder(indexFolderName)); From 2577b3838fc1fa663989b59633daf698ca6d82b8 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Thu, 15 Nov 2018 12:27:30 +0300 Subject: [PATCH 55/57] s/re-write/upgrade --- .../main/java/org/elasticsearch/gateway/GatewayMetaState.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index a3466ba25e5ce..b21fd5d3d85a5 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -137,7 +137,7 @@ private void upgradeMetaData(MetaDataIndexUpgradeService metaDataIndexUpgradeSer final Manifest newManifest = new Manifest(globalStateGeneration, indices); writer.writeManifestAndCleanup("startup", newManifest); } catch (Exception e) { - logger.error("failed to read or re-write local state, exiting...", e); + logger.error("failed to read or upgrade local state, exiting...", e); throw e; } } From 8b0ff4c828ec108fa9cce59c99923451f5b4d895 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Thu, 15 Nov 2018 12:28:11 +0300 Subject: [PATCH 56/57] typo dangling --- .../main/java/org/elasticsearch/gateway/GatewayMetaState.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index b21fd5d3d85a5..62c2c56acac19 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -223,7 +223,7 @@ long writeIndex(String reason, IndexMetaData metaData) throws WriteStateExceptio if (previousGeneration != null) { // we prefer not to clean-up index metadata in case of rollback, // if it's not referenced by previous manifest file - // not to break dandling indices functionality + // not to break dangling indices functionality rollbackCleanupActions.add(() -> metaStateService.cleanupIndex(index, previousGeneration)); } long generation = metaStateService.writeIndex(reason, metaData); From 21cec41423af1cb88620defd68b184007e129c70 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Thu, 15 Nov 2018 12:28:43 +0300 Subject: [PATCH 57/57] Rename test --- .../java/org/elasticsearch/gateway/GatewayMetaStateTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java index b9cf33f0d5156..901b70f7f7449 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java @@ -374,7 +374,7 @@ private static MetaData randomMetaDataForTx() { return builder.build(); } - public void testTransactionAtomicityWithFailures() throws IOException { + public void testAtomicityWithFailures() throws IOException { try (NodeEnvironment env = newNodeEnvironment()) { MetaStateServiceWithFailures metaStateService = new MetaStateServiceWithFailures(randomIntBetween(100, 1000), env, xContentRegistry());