From 595ff872873dab8db7c4c3c68b7a3629ddf02d6e Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 12 Feb 2019 14:56:27 +0100 Subject: [PATCH 1/4] Handle the fact that `ShardStats` instance may have no commit or seqno stats The should fix the following NPE: ``` [2019-02-11T23:27:48,452][WARN ][o.e.p.PersistentTasksNodeService] [node_s_0] task kD8YzUhHTK6uKNBNQI-1ZQ-0 failed with an exception 1> java.lang.NullPointerException: null 1> at org.elasticsearch.xpack.ccr.action.ShardFollowTasksExecutor.lambda$fetchFollowerShardInfo$7(ShardFollowTasksExecutor.java:305) ~[main/:?] 1> at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:61) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] 1> at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:68) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] 1> at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:64) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] 1> at org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$AsyncAction.onCompletion(TransportBroadcastByNodeAction.java:383) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] 1> at org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$AsyncAction.onNodeResponse(TransportBroadcastByNodeAction.java:352) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] 1> at org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$AsyncAction$1.handleResponse(TransportBroadcastByNodeAction.java:324) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] 1> at org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$AsyncAction$1.handleResponse(TransportBroadcastByNodeAction.java:314) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] 1> at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1108) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] 1> at org.elasticsearch.transport.TransportService$DirectResponseChannel.processResponse(TransportService.java:1189) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] 1> at org.elasticsearch.transport.TransportService$DirectResponseChannel.sendResponse(TransportService.java:1169) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] 1> at org.elasticsearch.transport.TaskTransportChannel.sendResponse(TaskTransportChannel.java:54) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] 1> at org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:417) [elasticsearch-8.0.0-SNAP SHOT.jar:8.0.0-SNAPSHOT] 1> at org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:391) [elasticsearch-8.0.0-SNAP SHOT.jar:8.0.0-SNAPSHOT] 1> at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] 1> at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:687) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] 1> at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] 1> at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT] 1> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_202] 1> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_202] 1> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_202] ``` Relates to #38779 --- .../ccr/action/ShardFollowTasksExecutor.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java index c0e2d7f54b318..bd75e61dfc4f5 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java @@ -9,6 +9,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.UnavailableShardsException; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; @@ -302,9 +303,19 @@ private void fetchFollowerShardInfo( if (filteredShardStats.isPresent()) { final ShardStats shardStats = filteredShardStats.get(); final CommitStats commitStats = shardStats.getCommitStats(); - final String historyUUID = commitStats.getUserData().get(Engine.HISTORY_UUID_KEY); - + if (commitStats == null) { + // UnavailableShardsException will be retried byShardFollowNodeTask.shouldRetry(...) + errorHandler.accept(new UnavailableShardsException(shardId, "commit_stats are missing")); + return; + } final SeqNoStats seqNoStats = shardStats.getSeqNoStats(); + if (seqNoStats == null) { + // UnavailableShardsException will be retried byShardFollowNodeTask.shouldRetry(...) + errorHandler.accept(new UnavailableShardsException(shardId, "seq_no_stats are missing")); + return; + } + + final String historyUUID = commitStats.getUserData().get(Engine.HISTORY_UUID_KEY); final long globalCheckpoint = seqNoStats.getGlobalCheckpoint(); final long maxSeqNo = seqNoStats.getMaxSeqNo(); handler.accept(historyUUID, globalCheckpoint, maxSeqNo); From a04b1e01ad77acfd71a67e14e8176c5a3f435309 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 12 Feb 2019 16:57:24 +0100 Subject: [PATCH 2/4] unmuted test --- .../src/test/java/org/elasticsearch/xpack/ccr/FollowStatsIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/FollowStatsIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/FollowStatsIT.java index e6662f3770d24..1f1c6cd5c64e3 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/FollowStatsIT.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/FollowStatsIT.java @@ -149,7 +149,6 @@ public void testFollowStatsApiResourceNotFound() throws Exception { assertAcked(client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request("follower1")).actionGet()); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/38779") public void testFollowStatsApiIncludeShardFollowStatsWithRemovedFollowerIndex() throws Exception { final String leaderIndexSettings = getIndexSettings(1, 0, singletonMap(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true")); From d4c3ddd1b20b23d727c16320cca06a286672567d Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 12 Feb 2019 17:24:40 +0100 Subject: [PATCH 3/4] AlreadyClosedException --- .../xpack/ccr/action/ShardFollowTasksExecutor.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java index bd75e61dfc4f5..a1ab64d88ffbf 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java @@ -8,8 +8,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.lucene.store.AlreadyClosedException; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.UnavailableShardsException; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; @@ -305,13 +305,13 @@ private void fetchFollowerShardInfo( final CommitStats commitStats = shardStats.getCommitStats(); if (commitStats == null) { // UnavailableShardsException will be retried byShardFollowNodeTask.shouldRetry(...) - errorHandler.accept(new UnavailableShardsException(shardId, "commit_stats are missing")); + errorHandler.accept(new AlreadyClosedException(shardId + " commit_stats are missing")); return; } final SeqNoStats seqNoStats = shardStats.getSeqNoStats(); if (seqNoStats == null) { // UnavailableShardsException will be retried byShardFollowNodeTask.shouldRetry(...) - errorHandler.accept(new UnavailableShardsException(shardId, "seq_no_stats are missing")); + errorHandler.accept(new AlreadyClosedException(shardId + " seq_no_stats are missing")); return; } From d5968fcbfe8db468eb5c1b07585ccb73c7ea843c Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 12 Feb 2019 17:35:31 +0100 Subject: [PATCH 4/4] updated comment --- .../xpack/ccr/action/ShardFollowTasksExecutor.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java index a1ab64d88ffbf..d9af0d6e71c04 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java @@ -304,13 +304,15 @@ private void fetchFollowerShardInfo( final ShardStats shardStats = filteredShardStats.get(); final CommitStats commitStats = shardStats.getCommitStats(); if (commitStats == null) { - // UnavailableShardsException will be retried byShardFollowNodeTask.shouldRetry(...) + // If commitStats is null then AlreadyClosedException has been thrown: TransportIndicesStatsAction#shardOperation(...) + // AlreadyClosedException will be retried byShardFollowNodeTask.shouldRetry(...) errorHandler.accept(new AlreadyClosedException(shardId + " commit_stats are missing")); return; } final SeqNoStats seqNoStats = shardStats.getSeqNoStats(); if (seqNoStats == null) { - // UnavailableShardsException will be retried byShardFollowNodeTask.shouldRetry(...) + // If seqNoStats is null then AlreadyClosedException has been thrown at TransportIndicesStatsAction#shardOperation(...) + // AlreadyClosedException will be retried byShardFollowNodeTask.shouldRetry(...) errorHandler.accept(new AlreadyClosedException(shardId + " seq_no_stats are missing")); return; }