From bb565df5920ed1fcef56100bbc0fe77afeb2bf1d Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Thu, 27 Dec 2018 10:04:53 +0100 Subject: [PATCH 1/9] Add support for cluster alias to SearchRequest With the upcoming cross-cluster search alternate execution mode, the CCS node will be able to split a CCS request into multiple search requests, one per remote cluster involved. In order to do that, the CCS node has to be able to signal to each remote cluster that such sub-requests are part of a CCS request. Each cluster does not know about the other clusters involved, and does not know either what alias it is given in the CCS node, hence the CCS coordinating node needs to be able to provide the alias as part of the search request so that it is used as index prefix in the returned search hits. The cluster alias is a notion that's already supported in the search shards iterator and search shard target, but it is currently used in CCS as both index prefix and connection lookup key when fanning out to all the shards. With CCS alternate execution mode the provided cluster alias needs to be used only as index prefix, as shards are local to each cluster hence no cluster alias should be used for connection lookups. Such distinction is introduced through a new class called CCSInfo that replaces the previous cluster alias optional string and allows to set the cluster alias as well as a boolean flag that signals the current CCS execution mode. The cluster alias can be set to the SearchRequest at the transport layer only, and its getter/setter methods are package private. Relates to #32125 --- .../search/AbstractSearchAsyncAction.java | 3 +- .../search/CanMatchPreFilterSearchPhase.java | 2 +- .../action/search/DfsQueryPhase.java | 2 +- .../action/search/FetchSearchPhase.java | 5 +- .../action/search/InitialSearchPhase.java | 7 +- .../action/search/ScrollIdForNode.java | 1 + .../SearchDfsQueryThenFetchAsyncAction.java | 2 +- .../SearchQueryThenFetchAsyncAction.java | 2 +- .../action/search/SearchRequest.java | 35 ++++- .../search/SearchScrollAsyncAction.java | 6 +- ...SearchScrollQueryThenFetchAsyncAction.java | 4 +- .../action/search/SearchShardIterator.java | 44 +++++- .../action/search/ShardSearchFailure.java | 16 +- .../action/search/TransportSearchAction.java | 10 +- .../action/search/TransportSearchHelper.java | 6 +- .../org/elasticsearch/search/CCSInfo.java | 137 ++++++++++++++++++ .../search/DefaultSearchContext.java | 4 +- .../org/elasticsearch/search/SearchHit.java | 4 +- .../elasticsearch/search/SearchService.java | 7 +- .../search/SearchShardTarget.java | 61 +++++--- .../internal/ShardSearchLocalRequest.java | 19 +-- .../search/internal/ShardSearchRequest.java | 8 +- .../internal/ShardSearchTransportRequest.java | 12 +- .../elasticsearch/ExceptionsHelperTests.java | 4 +- .../AbstractSearchAsyncActionTests.java | 11 +- .../search/SearchPhaseControllerTests.java | 27 +++- .../search/SearchRequestTests.java | 50 +++++-- .../action/search/SearchResponseTests.java | 2 +- .../search/SearchShardIteratorTests.java | 78 ++++++++++ .../search/ShardSearchFailureTests.java | 40 ++++- .../search/TransportSearchActionTests.java | 43 +++++- .../search/TransportSearchHelperTests.java | 8 +- .../index/SearchSlowLogTests.java | 5 +- .../rest/action/RestActionsTests.java | 4 +- .../search/DefaultSearchContextTests.java | 17 ++- .../elasticsearch/search/SearchHitTests.java | 61 ++++++-- .../elasticsearch/search/SearchHitsTests.java | 13 +- .../search/SearchServiceTests.java | 40 ++++- .../ShardSearchTransportRequestTests.java | 72 +++++---- .../search/slice/SliceBuilderTests.java | 3 +- 40 files changed, 691 insertions(+), 184 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/search/CCSInfo.java rename server/src/test/java/org/elasticsearch/{ => action}/search/SearchRequestTests.java (81%) create mode 100644 server/src/test/java/org/elasticsearch/action/search/SearchShardIteratorTests.java diff --git a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java index 27293e8e50f8d..fccbf9f6b2354 100644 --- a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java @@ -319,7 +319,6 @@ public final void onFailure(Exception e) { } public final ShardSearchTransportRequest buildShardSearchRequest(SearchShardIterator shardIt) { - String clusterAlias = shardIt.getClusterAlias(); AliasFilter filter = aliasFilter.get(shardIt.shardId().getIndex().getUUID()); assert filter != null; float indexBoost = concreteIndexBoosts.getOrDefault(shardIt.shardId().getIndex().getUUID(), DEFAULT_INDEX_BOOST); @@ -327,7 +326,7 @@ public final ShardSearchTransportRequest buildShardSearchRequest(SearchShardIter final String[] routings = indexRoutings.getOrDefault(indexName, Collections.emptySet()) .toArray(new String[0]); return new ShardSearchTransportRequest(shardIt.getOriginalIndices(), request, shardIt.shardId(), getNumShards(), - filter, indexBoost, timeProvider.getAbsoluteStartMillis(), clusterAlias, routings); + filter, indexBoost, timeProvider.getAbsoluteStartMillis(), shardIt.getCCSInfo(), routings); } /** diff --git a/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java index bea5a975b2840..cc3552eb09378 100644 --- a/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java @@ -66,7 +66,7 @@ final class CanMatchPreFilterSearchPhase extends AbstractSearchAsyncAction listener) { - getSearchTransport().sendCanMatch(getConnection(shardIt.getClusterAlias(), shard.currentNodeId()), + getSearchTransport().sendCanMatch(getConnection(shardIt.getConnectionAlias(), shard.currentNodeId()), buildShardSearchRequest(shardIt), getTask(), listener); } diff --git a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java index 1d8d702520e4c..17a1d399d7a2c 100644 --- a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java @@ -71,7 +71,7 @@ public void run() throws IOException { () -> context.executeNextPhase(this, nextPhaseFactory.apply(queryResult)), context); for (final DfsSearchResult dfsResult : resultList) { final SearchShardTarget searchShardTarget = dfsResult.getSearchShardTarget(); - Transport.Connection connection = context.getConnection(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId()); + Transport.Connection connection = context.getConnection(searchShardTarget.getConnectionAlias(), searchShardTarget.getNodeId()); QuerySearchRequest querySearchRequest = new QuerySearchRequest(searchShardTarget.getOriginalIndices(), dfsResult.getRequestId(), dfs); final int shardIndex = dfsResult.getShardIndex(); diff --git a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java index 1bbca35cb9a54..1bf993b6f4c39 100644 --- a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java @@ -137,7 +137,7 @@ private void innerRun() throws IOException { counter.countDown(); } else { SearchShardTarget searchShardTarget = queryResult.getSearchShardTarget(); - Transport.Connection connection = context.getConnection(searchShardTarget.getClusterAlias(), + Transport.Connection connection = context.getConnection(searchShardTarget.getConnectionAlias(), searchShardTarget.getNodeId()); ShardFetchSearchRequest fetchSearchRequest = createFetchRequest(queryResult.queryResult().getRequestId(), i, entry, lastEmittedDocPerShard, searchShardTarget.getOriginalIndices()); @@ -190,7 +190,8 @@ private void releaseIrrelevantSearchContext(QuerySearchResult queryResult) { if (context.getRequest().scroll() == null && queryResult.hasSearchContext()) { try { SearchShardTarget searchShardTarget = queryResult.getSearchShardTarget(); - Transport.Connection connection = context.getConnection(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId()); + Transport.Connection connection = context.getConnection(searchShardTarget.getConnectionAlias(), + searchShardTarget.getNodeId()); context.sendReleaseSearchContext(queryResult.getRequestId(), connection, searchShardTarget.getOriginalIndices()); } catch (Exception e) { context.getLogger().trace("failed to release context", e); diff --git a/server/src/main/java/org/elasticsearch/action/search/InitialSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/InitialSearchPhase.java index 4e0db4644786e..30ad0529f5c34 100644 --- a/server/src/main/java/org/elasticsearch/action/search/InitialSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/InitialSearchPhase.java @@ -90,8 +90,7 @@ private void onShardFailure(final int shardIndex, @Nullable ShardRouting shard, final SearchShardIterator shardIt, Exception e) { // we always add the shard failure for a specific shard instance // we do make sure to clean it on a successful response from a shard - SearchShardTarget shardTarget = new SearchShardTarget(nodeId, shardIt.shardId(), shardIt.getClusterAlias(), - shardIt.getOriginalIndices()); + SearchShardTarget shardTarget = shardIt.newSearchShardTarget(nodeId); onShardFailure(shardIndex, shardTarget, e); if (totalOps.incrementAndGet() == expectedTotalOps) { @@ -257,8 +256,8 @@ private void performPhaseOnShard(final int shardIndex, final SearchShardIterator Runnable r = () -> { final Thread thread = Thread.currentThread(); try { - executePhaseOnShard(shardIt, shard, new SearchActionListener(new SearchShardTarget(shard.currentNodeId(), - shardIt.shardId(), shardIt.getClusterAlias(), shardIt.getOriginalIndices()), shardIndex) { + executePhaseOnShard(shardIt, shard, new SearchActionListener( + shardIt.newSearchShardTarget(shard.currentNodeId()), shardIndex) { @Override public void innerOnResponse(FirstResult result) { try { diff --git a/server/src/main/java/org/elasticsearch/action/search/ScrollIdForNode.java b/server/src/main/java/org/elasticsearch/action/search/ScrollIdForNode.java index fc6585054ddf9..18b61516897d1 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ScrollIdForNode.java +++ b/server/src/main/java/org/elasticsearch/action/search/ScrollIdForNode.java @@ -36,6 +36,7 @@ public String getNode() { return node; } + @Nullable public String getClusterAlias() { return clusterAlias; } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java index 0782fbb310b65..e06522a367b1c 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java @@ -53,7 +53,7 @@ final class SearchDfsQueryThenFetchAsyncAction extends AbstractSearchAsyncAction @Override protected void executePhaseOnShard(final SearchShardIterator shardIt, final ShardRouting shard, final SearchActionListener listener) { - getSearchTransport().sendExecuteDfs(getConnection(shardIt.getClusterAlias(), shard.currentNodeId()), + getSearchTransport().sendExecuteDfs(getConnection(shardIt.getConnectionAlias(), shard.currentNodeId()), buildShardSearchRequest(shardIt) , getTask(), listener); } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java index bbd84011de00b..04f1f281cdad7 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java @@ -51,7 +51,7 @@ final class SearchQueryThenFetchAsyncAction extends AbstractSearchAsyncAction listener) { - getSearchTransport().sendExecuteQuery(getConnection(shardIt.getClusterAlias(), shard.currentNodeId()), + getSearchTransport().sendExecuteQuery(getConnection(shardIt.getConnectionAlias(), shard.currentNodeId()), buildShardSearchRequest(shardIt), getTask(), listener); } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java index 68968c071f43c..f5e0eac8e2681 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -91,6 +91,8 @@ public final class SearchRequest extends ActionRequest implements IndicesRequest private IndicesOptions indicesOptions = DEFAULT_INDICES_OPTIONS; + private String clusterAlias; + public SearchRequest() { } @@ -111,6 +113,7 @@ public SearchRequest(SearchRequest searchRequest) { this.searchType = searchRequest.searchType; this.source = searchRequest.source; this.types = searchRequest.types; + this.clusterAlias = searchRequest.clusterAlias; } /** @@ -158,6 +161,10 @@ public SearchRequest(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_6_3_0)) { allowPartialSearchResults = in.readOptionalBoolean(); } + //TODO update version after backport + if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + clusterAlias = in.readOptionalString(); + } } @Override @@ -181,6 +188,10 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_6_3_0)) { out.writeOptionalBoolean(allowPartialSearchResults); } + //TODO update version after backport + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + out.writeOptionalString(clusterAlias); + } } @Override @@ -209,6 +220,24 @@ public ActionRequestValidationException validate() { return validationException; } + /** + * Returns the alias of the cluster that this search request is being executed on. A non-null value indicates that this search request + * is being executed as part of a locally reduced cross-cluster search request. The cluster alias is used to provide the cluster alias + * to prefix returned index names with. + */ + @Nullable + String getClusterAlias() { + return clusterAlias; + } + + /** + * Sets the cluster alias for this search request. Used when a {@link SearchRequest} is executed as part of a cross-cluster search + * request performing local reduction on each cluster. This way the coordinating CCS node provides the alias to prefix index names with. + */ + void setClusterAlias(String clusterAlias) { + this.clusterAlias = clusterAlias; + } + /** * Sets the indices the search will be executed on. */ @@ -529,14 +558,15 @@ public boolean equals(Object o) { Objects.equals(maxConcurrentShardRequests, that.maxConcurrentShardRequests) && Objects.equals(preFilterShardSize, that.preFilterShardSize) && Objects.equals(indicesOptions, that.indicesOptions) && - Objects.equals(allowPartialSearchResults, that.allowPartialSearchResults); + Objects.equals(allowPartialSearchResults, that.allowPartialSearchResults) && + Objects.equals(clusterAlias, that.clusterAlias); } @Override public int hashCode() { return Objects.hash(searchType, Arrays.hashCode(indices), routing, preference, source, requestCache, scroll, Arrays.hashCode(types), indicesOptions, batchedReduceSize, maxConcurrentShardRequests, preFilterShardSize, - allowPartialSearchResults); + allowPartialSearchResults, clusterAlias); } @Override @@ -554,6 +584,7 @@ public String toString() { ", batchedReduceSize=" + batchedReduceSize + ", preFilterShardSize=" + preFilterShardSize + ", allowPartialSearchResults=" + allowPartialSearchResults + + ", clusterAlias=" + clusterAlias + ", source=" + source + '}'; } } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java index 81d7e66d19edb..f83393d759db2 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.common.util.concurrent.CountDown; +import org.elasticsearch.search.CCSInfo; import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.internal.InternalScrollSearchRequest; @@ -173,8 +174,11 @@ protected void setSearchShardTarget(T response) { // re-create the search target and add the cluster alias if there is any, // we need this down the road for subseq. phases SearchShardTarget searchShardTarget = response.getSearchShardTarget(); + //We can assume that we are executing as part of a CCS request with a single coordination step. + //Scroll is not supported when performing multiple coordination steps in each cluster. + CCSInfo ccsInfo = new CCSInfo(target.getClusterAlias(), false); response.setSearchShardTarget(new SearchShardTarget(searchShardTarget.getNodeId(), searchShardTarget.getShardId(), - target.getClusterAlias(), null)); + ccsInfo, null)); } } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java index df18296de2a4a..326c8e494d8d5 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java @@ -89,9 +89,9 @@ public void run() { ShardFetchRequest shardFetchRequest = new ShardFetchRequest(querySearchResult.getRequestId(), docIds, lastEmittedDoc); SearchShardTarget searchShardTarget = querySearchResult.getSearchShardTarget(); - DiscoveryNode node = clusterNodeLookup.apply(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId()); + DiscoveryNode node = clusterNodeLookup.apply(searchShardTarget.getConnectionAlias(), searchShardTarget.getNodeId()); assert node != null : "target node is null in secondary phase"; - Transport.Connection connection = getConnection(searchShardTarget.getClusterAlias(), node); + Transport.Connection connection = getConnection(searchShardTarget.getConnectionAlias(), node); searchTransportService.sendExecuteFetchScroll(connection, shardFetchRequest, task, new SearchActionListener(querySearchResult.getSearchShardTarget(), index) { @Override diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java b/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java index c36d2b7908f78..e51f2f86dd18f 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java @@ -22,31 +22,39 @@ import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.cluster.routing.PlainShardIterator; import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.common.Nullable; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.search.CCSInfo; +import org.elasticsearch.search.SearchShardTarget; import java.util.List; /** * Extension of {@link PlainShardIterator} used in the search api, which also holds the {@link OriginalIndices} - * of the search request. Useful especially with cross cluster search, as each cluster has its own set of original indices. + * of the search request (useful especially with cross-cluster search, as each cluster has its own set of original indices) as well as + * information about the cross-cluster search execution. + * @see OriginalIndices + * @see CCSInfo */ public final class SearchShardIterator extends PlainShardIterator { private final OriginalIndices originalIndices; - private String clusterAlias; + private final CCSInfo ccsInfo; private boolean skip = false; /** * Creates a {@link PlainShardIterator} instance that iterates over a subset of the given shards * this the a given shardId. * + * @param ccsInfo information about cross-cluster search execution, null if not applicable * @param shardId shard id of the group * @param shards shards to iterate + * @param originalIndices the indices that the search request originally related to (before any rewriting happened) */ - public SearchShardIterator(String clusterAlias, ShardId shardId, List shards, OriginalIndices originalIndices) { + public SearchShardIterator(@Nullable CCSInfo ccsInfo, ShardId shardId, List shards, OriginalIndices originalIndices) { super(shardId, shards); this.originalIndices = originalIndices; - this.clusterAlias = clusterAlias; + this.ccsInfo = ccsInfo; } /** @@ -56,8 +64,32 @@ public OriginalIndices getOriginalIndices() { return originalIndices; } - public String getClusterAlias() { - return clusterAlias; + /** + * Returns the cluster alias needed to lookup the connection when sending shard level requests in the context of this iterator. + * null indicates that the shard is local, meaning that either we are not executing a cross-cluster search request, + * or we are but each cluster performs its own reduction. + */ + @Nullable + public String getConnectionAlias() { + return ccsInfo == null ? null : ccsInfo.getConnectionAlias(); + } + + /** + * Returns info about cross-cluster search execution if applicable. + * null means that we are not executing a cross-cluster search request. + * @see CCSInfo + */ + @Nullable + public CCSInfo getCCSInfo() { + return ccsInfo; + } + + /** + * Creates a new shard target from this iterator, pointing at the node identified by the provided identifier. + * @see SearchShardTarget + */ + SearchShardTarget newSearchShardTarget(String nodeId) { + return new SearchShardTarget(nodeId, shardId(), ccsInfo, originalIndices); } /** diff --git a/server/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java b/server/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java index ddfadfa57e31e..6546a9ddb2a76 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java +++ b/server/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java @@ -32,6 +32,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.CCSInfo; import org.elasticsearch.search.SearchException; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.transport.RemoteClusterAware; @@ -98,8 +99,8 @@ public static ShardSearchFailure readShardSearchFailure(StreamInput in) throws I @Override public void readFrom(StreamInput in) throws IOException { - if (in.readBoolean()) { - shardTarget = new SearchShardTarget(in); + shardTarget = in.readOptionalWriteable(SearchShardTarget::new); + if (shardTarget != null) { index = shardTarget.getFullyQualifiedIndexName(); shardId = shardTarget.getShardId().getId(); } @@ -110,12 +111,7 @@ public void readFrom(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { - if (shardTarget == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - shardTarget.writeTo(out); - } + out.writeOptionalWriteable(shardTarget); out.writeString(reason); RestStatus.writeTo(out, status); out.writeException(cause); @@ -174,8 +170,8 @@ public static ShardSearchFailure fromXContent(XContentParser parser) throws IOEx } SearchShardTarget searchShardTarget = null; if (nodeId != null) { - searchShardTarget = new SearchShardTarget(nodeId, - new ShardId(new Index(indexName, IndexMetaData.INDEX_UUID_NA_VALUE), shardId), clusterAlias, OriginalIndices.NONE); + searchShardTarget = new SearchShardTarget(nodeId, new ShardId(new Index(indexName, IndexMetaData.INDEX_UUID_NA_VALUE), shardId), + CCSInfo.fromClusterAlias(clusterAlias), OriginalIndices.NONE); } return new ShardSearchFailure(exception, searchShardTarget); } diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index 4c1a953965fcd..06e6fe4d6989e 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -34,6 +34,7 @@ import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.routing.ShardIterator; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Setting; @@ -41,6 +42,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.query.Rewriteable; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.search.CCSInfo; import org.elasticsearch.search.SearchService; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.internal.AliasFilter; @@ -262,7 +264,7 @@ static BiFunction processRemoteShards(Map localShardsIterator = clusterService.operationRouting().searchShards(clusterState, concreteIndices, routingMap, searchRequest.preference(), searchService.getResponseCollectorService(), nodeSearchCounts); GroupShardsIterator shardIterators = mergeShardsIterators(localShardsIterator, localIndices, - remoteShardIterators); + searchRequest.getClusterAlias(), remoteShardIterators); failIfOverShardCountLimit(clusterService, shardIterators.size()); @@ -359,10 +361,12 @@ private boolean shouldPreFilterSearchShards(SearchRequest searchRequest, GroupSh static GroupShardsIterator mergeShardsIterators(GroupShardsIterator localShardsIterator, OriginalIndices localIndices, + @Nullable String localClusterAlias, List remoteShardIterators) { + CCSInfo localCCSInfo = localClusterAlias == null ? null : new CCSInfo(localClusterAlias, true); List shards = new ArrayList<>(remoteShardIterators); for (ShardIterator shardIterator : localShardsIterator) { - shards.add(new SearchShardIterator(null, shardIterator.shardId(), shardIterator.getShardRoutings(), localIndices)); + shards.add(new SearchShardIterator(localCCSInfo, shardIterator.shardId(), shardIterator.getShardRoutings(), localIndices)); } return new GroupShardsIterator<>(shards); } diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java index 7a0bb63478c76..d3c14f18ca9f5 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java @@ -43,9 +43,9 @@ static String buildScrollId(AtomicArray searchPhase for (SearchPhaseResult searchPhaseResult : searchPhaseResults.asList()) { out.writeLong(searchPhaseResult.getRequestId()); SearchShardTarget searchShardTarget = searchPhaseResult.getSearchShardTarget(); - if (searchShardTarget.getClusterAlias() != null) { - out.writeString(RemoteClusterAware.buildRemoteIndexName(searchShardTarget.getClusterAlias(), - searchShardTarget.getNodeId())); + String hitIndexPrefix = searchShardTarget.getHitIndexPrefix(); + if (hitIndexPrefix != null) { + out.writeString(RemoteClusterAware.buildRemoteIndexName(hitIndexPrefix, searchShardTarget.getNodeId())); } else { out.writeString(searchShardTarget.getNodeId()); } diff --git a/server/src/main/java/org/elasticsearch/search/CCSInfo.java b/server/src/main/java/org/elasticsearch/search/CCSInfo.java new file mode 100644 index 0000000000000..631b7dba5ce5f --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/CCSInfo.java @@ -0,0 +1,137 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search; + +import org.elasticsearch.Version; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; + +import java.io.IOException; +import java.util.Objects; + +/** + * Encloses information about a cross-cluster search request: the cluster alias that will be used to prefix the index name returned + * with the {@link SearchHit}, and whether the sub-request is part of a locally reduced cross-cluster search request. + * In case a cross-cluster search request fans out to all the (local and remote) shards in one go, the cluster alias has to be used + * as both the index prefix and the identifier for {@link org.elasticsearch.transport.Transport.Connection} lookup when sending each + * shard level request. On the other hand, when reducing results locally in each remote cluster, the cluster alias has to be used only + * as the index prefix, all the targeted shards in each sub-search request are local. + */ +public final class CCSInfo implements Writeable { + private final String clusterAlias; + private final boolean localReduction; + + public CCSInfo(String clusterAlias, boolean localReduction) { + this.clusterAlias = Objects.requireNonNull(clusterAlias); + this.localReduction = localReduction; + } + + private CCSInfo(StreamInput in) throws IOException { + this.clusterAlias = in.readString(); + this.localReduction = in.readBoolean(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(clusterAlias); + out.writeBoolean(localReduction); + } + + /** + * Returns the prefix to be used for the index names of each search result, to indicate which cluster returned it. + */ + public String getHitIndexPrefix() { + return clusterAlias; + } + + /** + * Returns the cluster alias needed to lookup the connection when sending shard level requests. null indicates that the + * shards are local, hence we are executing a cross-cluster search request as part of which each cluster performs its own reduction. + */ + @Nullable + public String getConnectionAlias() { + return localReduction ? null : clusterAlias; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + CCSInfo ccsInfo = (CCSInfo) o; + return localReduction == ccsInfo.localReduction && + Objects.equals(clusterAlias, ccsInfo.clusterAlias); + } + + @Override + public int hashCode() { + return Objects.hash(clusterAlias, localReduction); + } + + /** + * Reads an optional {@link CCSInfo} instance from the provided {@link StreamInput}. + * Handles backwards compatibility with previous versions that supported only the cluster alias as an optional string. In such + * cases the localReduction flag will be set to false as CCS local reductions were not supported anyways. + */ + @Nullable + public static CCSInfo read(StreamInput in) throws IOException { + //TODO update version once backported + if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + return in.readOptionalWriteable(CCSInfo::new); + } + if (in.readBoolean()) { + return new CCSInfo(in.readString(), false); + } + return null; + } + + /** + * Writes an optional {@link CCSInfo} instance to the provided {@link StreamOutput}. + * Handles backwards compatibility with previous versions that supported only the cluster alias as an optional string. In such + * cases the localReduction flag will be set to false as CCS local reductions were not supported anyways. + */ + public static void write(@Nullable CCSInfo ccsInfo, StreamOutput out) throws IOException { + //TODO update version once backported + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + out.writeOptionalWriteable(ccsInfo); + } else { + if (ccsInfo == null) { + out.writeBoolean(false); + } else { + out.writeOptionalString(ccsInfo.clusterAlias); + } + } + } + + /** + * Returns a new {@link CCSInfo} instance or null depending on the provided cluster alias. + * This method is used as a central place to build a {@link CCSInfo} when the cluster alias may be null, hence null + * should be returned, and when the localReduction flag is not available (as it's not serialized at REST) hence false will be assumed. + */ + @Nullable + public static CCSInfo fromClusterAlias(@Nullable String clusterAlias) { + return clusterAlias == null ? null : new CCSInfo(clusterAlias, false); + } +} diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 091fd5f8c85e0..4d4d3323d086d 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -159,7 +159,7 @@ final class DefaultSearchContext extends SearchContext { DefaultSearchContext(long id, ShardSearchRequest request, SearchShardTarget shardTarget, Engine.Searcher engineSearcher, ClusterService clusterService, IndexService indexService, IndexShard indexShard, BigArrays bigArrays, Counter timeEstimateCounter, TimeValue timeout, - FetchPhase fetchPhase, String clusterAlias, Version minNodeVersion) { + FetchPhase fetchPhase, Version minNodeVersion) { this.id = id; this.request = request; this.fetchPhase = fetchPhase; @@ -179,7 +179,7 @@ final class DefaultSearchContext extends SearchContext { this.timeout = timeout; this.minNodeVersion = minNodeVersion; queryShardContext = indexService.newQueryShardContext(request.shardId().id(), searcher.getIndexReader(), request::nowInMillis, - clusterAlias); + shardTarget.getHitIndexPrefix()); queryShardContext.setTypes(request.types()); queryBoost = request.indexBoost(); } diff --git a/server/src/main/java/org/elasticsearch/search/SearchHit.java b/server/src/main/java/org/elasticsearch/search/SearchHit.java index 7fd68852ce284..3a87505cb1965 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchHit.java +++ b/server/src/main/java/org/elasticsearch/search/SearchHit.java @@ -354,7 +354,7 @@ public void shard(SearchShardTarget target) { this.shard = target; if (target != null) { this.index = target.getIndex(); - this.clusterAlias = target.getClusterAlias(); + this.clusterAlias = target.getHitIndexPrefix(); } } @@ -578,7 +578,7 @@ public static SearchHit createFromMap(Map values) { String nodeId = get(Fields._NODE, values, null); if (shardId != null && nodeId != null) { assert shardId.getIndexName().equals(index); - searchHit.shard(new SearchShardTarget(nodeId, shardId, clusterAlias, OriginalIndices.NONE)); + searchHit.shard(new SearchShardTarget(nodeId, shardId, CCSInfo.fromClusterAlias(clusterAlias), OriginalIndices.NONE)); } else { //these fields get set anyways when setting the shard target, //but we set them explicitly when we don't have enough info to rebuild the shard target diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index 98f2e1d2e7ecf..2c6f9a0d594ed 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -668,8 +668,7 @@ final SearchContext createContext(ShardSearchRequest request) throws IOException return context; } - public DefaultSearchContext createSearchContext(ShardSearchRequest request, TimeValue timeout) - throws IOException { + public DefaultSearchContext createSearchContext(ShardSearchRequest request, TimeValue timeout) throws IOException { return createSearchContext(request, timeout, true, "search"); } @@ -679,12 +678,12 @@ private DefaultSearchContext createSearchContext(ShardSearchRequest request, Tim IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex()); IndexShard indexShard = indexService.getShard(request.shardId().getId()); SearchShardTarget shardTarget = new SearchShardTarget(clusterService.localNode().getId(), - indexShard.shardId(), request.getClusterAlias(), OriginalIndices.NONE); + indexShard.shardId(), request.getCCSInfo(), OriginalIndices.NONE); Engine.Searcher engineSearcher = indexShard.acquireSearcher(source); final DefaultSearchContext searchContext = new DefaultSearchContext(idGenerator.incrementAndGet(), request, shardTarget, engineSearcher, clusterService, indexService, indexShard, bigArrays, threadPool.estimatedTimeInMillisCounter(), timeout, - fetchPhase, request.getClusterAlias(), clusterService.state().nodes().getMinNodeVersion()); + fetchPhase, clusterService.state().nodes().getMinNodeVersion()); boolean success = false; try { // we clone the query shard context here just for rewriting otherwise we diff --git a/server/src/main/java/org/elasticsearch/search/SearchShardTarget.java b/server/src/main/java/org/elasticsearch/search/SearchShardTarget.java index 4a46c7202d14e..2301a29b5c153 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchShardTarget.java +++ b/server/src/main/java/org/elasticsearch/search/SearchShardTarget.java @@ -30,6 +30,7 @@ import org.elasticsearch.transport.RemoteClusterAware; import java.io.IOException; +import java.util.Objects; /** * The target that the search request was executed on. @@ -41,7 +42,7 @@ public final class SearchShardTarget implements Writeable, Comparablenull means that no prefix is needed, meaning that we are not executing a cross-cluster search request. + */ + @Nullable + public String getHitIndexPrefix() { + return ccsInfo == null ? null : ccsInfo.getHitIndexPrefix(); + } + + /** + * Returns the cluster alias needed to lookup the connection when sending shard level requests in the context of this shard target. + * null indicates that the shard is local, meaning that either we are not executing a cross-cluster search request, + * or we are but each cluster performs its own reduction. + */ + @Nullable + public String getConnectionAlias() { + return ccsInfo == null ? null : ccsInfo.getConnectionAlias(); } /** - * Returns the fully qualified index name, including the cluster alias. + * Returns the fully qualified index name, including the index prefix that indicates which cluster results come from. */ public String getFullyQualifiedIndexName() { - return RemoteClusterAware.buildRemoteIndexName(getClusterAlias(), getIndex()); + return RemoteClusterAware.buildRemoteIndexName(getHitIndexPrefix(), getIndex()); } @Override @@ -116,33 +132,32 @@ public void writeTo(StreamOutput out) throws IOException { out.writeText(nodeId); } shardId.writeTo(out); - out.writeOptionalString(clusterAlias); + CCSInfo.write(ccsInfo, out); } @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } SearchShardTarget that = (SearchShardTarget) o; - if (shardId.equals(that.shardId) == false) return false; - if (nodeId != null ? !nodeId.equals(that.nodeId) : that.nodeId != null) return false; - if (clusterAlias != null ? !clusterAlias.equals(that.clusterAlias) : that.clusterAlias != null) return false; - return true; + return Objects.equals(nodeId, that.nodeId) && + Objects.equals(shardId, that.shardId) && + Objects.equals(ccsInfo, that.ccsInfo); } @Override public int hashCode() { - int result = nodeId != null ? nodeId.hashCode() : 0; - result = 31 * result + (shardId.getIndexName() != null ? shardId.getIndexName().hashCode() : 0); - result = 31 * result + shardId.hashCode(); - result = 31 * result + (clusterAlias != null ? clusterAlias.hashCode() : 0); - return result; + return Objects.hash(nodeId, shardId, ccsInfo); } @Override public String toString() { - String shardToString = "[" + RemoteClusterAware.buildRemoteIndexName(clusterAlias, shardId.getIndexName()) + "][" + shardId.getId() - + "]"; + String shardToString = "[" + RemoteClusterAware.buildRemoteIndexName( + getHitIndexPrefix(), shardId.getIndexName()) + "][" + shardId.getId() + "]"; if (nodeId == null) { return "[_na_]" + shardToString; } diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java index 72a12b805eb17..83c101c5aceed 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java @@ -22,6 +22,7 @@ import org.elasticsearch.Version; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -31,6 +32,7 @@ import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.Rewriteable; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.search.CCSInfo; import org.elasticsearch.search.Scroll; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -57,7 +59,7 @@ */ public class ShardSearchLocalRequest implements ShardSearchRequest { - private String clusterAlias; + private CCSInfo ccsInfo; private ShardId shardId; private int numberOfShards; private SearchType searchType; @@ -76,8 +78,8 @@ public class ShardSearchLocalRequest implements ShardSearchRequest { ShardSearchLocalRequest() { } - ShardSearchLocalRequest(SearchRequest searchRequest, ShardId shardId, int numberOfShards, - AliasFilter aliasFilter, float indexBoost, long nowInMillis, String clusterAlias, String[] indexRoutings) { + ShardSearchLocalRequest(SearchRequest searchRequest, ShardId shardId, int numberOfShards, AliasFilter aliasFilter, float indexBoost, + long nowInMillis, @Nullable CCSInfo ccsInfo, String[] indexRoutings) { this(shardId, numberOfShards, searchRequest.searchType(), searchRequest.source(), searchRequest.types(), searchRequest.requestCache(), aliasFilter, indexBoost, searchRequest.allowPartialSearchResults(), indexRoutings, searchRequest.preference()); @@ -86,7 +88,7 @@ public class ShardSearchLocalRequest implements ShardSearchRequest { assert searchRequest.allowPartialSearchResults() != null; this.scroll = searchRequest.scroll(); this.nowInMillis = nowInMillis; - this.clusterAlias = clusterAlias; + this.ccsInfo = ccsInfo; } public ShardSearchLocalRequest(ShardId shardId, String[] types, long nowInMillis, AliasFilter aliasFilter) { @@ -113,7 +115,6 @@ public ShardSearchLocalRequest(ShardId shardId, int numberOfShards, SearchType s this.preference = preference; } - @Override public ShardId shardId() { return shardId; @@ -215,7 +216,7 @@ protected void innerReadFrom(StreamInput in) throws IOException { indexBoost = in.readFloat(); nowInMillis = in.readVLong(); requestCache = in.readOptionalBoolean(); - clusterAlias = in.readOptionalString(); + ccsInfo = CCSInfo.read(in); if (in.getVersion().onOrAfter(Version.V_6_3_0)) { allowPartialSearchResults = in.readOptionalBoolean(); } @@ -243,7 +244,7 @@ protected void innerWriteTo(StreamOutput out, boolean asKey) throws IOException out.writeVLong(nowInMillis); } out.writeOptionalBoolean(requestCache); - out.writeOptionalString(clusterAlias); + CCSInfo.write(ccsInfo, out); if (out.getVersion().onOrAfter(Version.V_6_3_0)) { out.writeOptionalBoolean(allowPartialSearchResults); } @@ -265,8 +266,8 @@ public BytesReference cacheKey() throws IOException { } @Override - public String getClusterAlias() { - return clusterAlias; + public CCSInfo getCCSInfo() { + return ccsInfo; } @Override diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java index 862c1a6960bed..d7e6123f95b5c 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java @@ -33,6 +33,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.AliasFilterParsingException; import org.elasticsearch.indices.InvalidAliasNameException; +import org.elasticsearch.search.CCSInfo; import org.elasticsearch.search.Scroll; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -151,10 +152,11 @@ static QueryBuilder parseAliasFilter(CheckedFunctionnull if the request if targeted to the local - * cluster. + * Returns information about cross-cluster search execution if this request is part of a cross-cluster search request, + * null otherwise + * @see CCSInfo */ - String getClusterAlias(); + CCSInfo getCCSInfo(); Rewriteable getRewriteable(); diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java index e7aad0bd51786..3436f5ed96a15 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.index.query.Rewriteable; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.search.CCSInfo; import org.elasticsearch.search.Scroll; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.tasks.Task; @@ -51,14 +52,11 @@ public class ShardSearchTransportRequest extends TransportRequest implements Sha private ShardSearchLocalRequest shardSearchLocalRequest; - public ShardSearchTransportRequest(){ - } - public ShardSearchTransportRequest(OriginalIndices originalIndices, SearchRequest searchRequest, ShardId shardId, int numberOfShards, AliasFilter aliasFilter, float indexBoost, long nowInMillis, - String clusterAlias, String[] indexRoutings) { + CCSInfo ccsInfo, String[] indexRoutings) { this.shardSearchLocalRequest = new ShardSearchLocalRequest(searchRequest, shardId, numberOfShards, aliasFilter, indexBoost, - nowInMillis, clusterAlias, indexRoutings); + nowInMillis, ccsInfo, indexRoutings); this.originalIndices = originalIndices; } @@ -203,8 +201,8 @@ public String getDescription() { } @Override - public String getClusterAlias() { - return shardSearchLocalRequest.getClusterAlias(); + public CCSInfo getCCSInfo() { + return shardSearchLocalRequest.getCCSInfo(); } @Override diff --git a/server/src/test/java/org/elasticsearch/ExceptionsHelperTests.java b/server/src/test/java/org/elasticsearch/ExceptionsHelperTests.java index 1d2a4ca6d5f75..d9f8996ccbe79 100644 --- a/server/src/test/java/org/elasticsearch/ExceptionsHelperTests.java +++ b/server/src/test/java/org/elasticsearch/ExceptionsHelperTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.CCSInfo; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.transport.RemoteClusterAware; @@ -136,8 +137,9 @@ private static ShardSearchFailure createShardFailureParsingException(String erro } private static SearchShardTarget createSearchShardTarget(String nodeId, int shardId, String index, String clusterAlias) { + CCSInfo ccsInfo = clusterAlias == null ? null : new CCSInfo(clusterAlias, randomBoolean()); return new SearchShardTarget(nodeId, - new ShardId(new Index(index, IndexMetaData.INDEX_UUID_NA_VALUE), shardId), clusterAlias, OriginalIndices.NONE); + new ShardId(new Index(index, IndexMetaData.INDEX_UUID_NA_VALUE), shardId), ccsInfo, OriginalIndices.NONE); } public void testGroupByNullTarget() { diff --git a/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java index 193878e2f5e04..84e21f7be464c 100644 --- a/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.search.CCSInfo; import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.search.internal.ShardSearchTransportRequest; @@ -116,7 +117,8 @@ private void runTestTook(final boolean controlled) { public void testBuildShardSearchTransportRequest() { final AtomicLong expected = new AtomicLong(); AbstractSearchAsyncAction action = createAction(false, expected); - SearchShardIterator iterator = new SearchShardIterator("test-cluster", new ShardId(new Index("name", "foo"), 1), + CCSInfo ccsInfo = randomBoolean() ? null : new CCSInfo(randomAlphaOfLengthBetween(5, 10), randomBoolean()); + SearchShardIterator iterator = new SearchShardIterator(ccsInfo, new ShardId(new Index("name", "foo"), 1), Collections.emptyList(), new OriginalIndices(new String[] {"name", "name1"}, IndicesOptions.strictExpand())); ShardSearchTransportRequest shardSearchTransportRequest = action.buildShardSearchRequest(iterator); assertEquals(IndicesOptions.strictExpand(), shardSearchTransportRequest.indicesOptions()); @@ -126,5 +128,12 @@ public void testBuildShardSearchTransportRequest() { assertArrayEquals(new String[] {"name", "name1"}, shardSearchTransportRequest.indices()); assertArrayEquals(new String[] {"bar", "baz"}, shardSearchTransportRequest.indexRoutings()); assertEquals("_shards:1,3", shardSearchTransportRequest.preference()); + assertSame(ccsInfo, shardSearchTransportRequest.getCCSInfo()); + if (ccsInfo == null) { + assertNull(shardSearchTransportRequest.getCCSInfo()); + } else { + assertEquals(ccsInfo.getHitIndexPrefix(), shardSearchTransportRequest.getCCSInfo().getHitIndexPrefix()); + assertEquals(ccsInfo.getConnectionAlias(), shardSearchTransportRequest.getCCSInfo().getConnectionAlias()); + } } } diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java index 4a8afe22b18aa..3170e17f618ac 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java @@ -29,12 +29,15 @@ import org.apache.lucene.search.TotalHits.Relation; import org.apache.lucene.search.grouping.CollapseTopFieldDocs; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.index.Index; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.search.CCSInfo; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; @@ -157,14 +160,18 @@ public void testMerge() { for (boolean trackTotalHits : new boolean[] {true, false}) { SearchPhaseController.ReducedQueryPhase reducedQueryPhase = searchPhaseController.reducedQueryPhase(queryResults.asList(), false, trackTotalHits); - AtomicArray searchPhaseResultAtomicArray = generateFetchResults(nShards, + AtomicArray fetchResults = generateFetchResults(nShards, reducedQueryPhase.sortedTopDocs.scoreDocs, reducedQueryPhase.suggest); InternalSearchResponse mergedResponse = searchPhaseController.merge(false, reducedQueryPhase, - searchPhaseResultAtomicArray.asList(), searchPhaseResultAtomicArray::get); + fetchResults.asList(), fetchResults::get); if (trackTotalHits == false) { assertNull(mergedResponse.hits.getTotalHits()); } + for (SearchHit hit : mergedResponse.hits().getHits()) { + SearchPhaseResult searchPhaseResult = fetchResults.get(hit.getShard().getShardId().id()); + assertSame(searchPhaseResult.getSearchShardTarget(), hit.getShard()); + } int suggestSize = 0; for (Suggest.Suggestion s : reducedQueryPhase.suggest) { Stream stream = s.getEntries().stream(); @@ -182,6 +189,8 @@ public void testMerge() { assertThat(options.size(), equalTo(suggestion.getEntries().get(0).getOptions().size())); for (CompletionSuggestion.Entry.Option option : options) { assertNotNull(option.getHit()); + SearchPhaseResult searchPhaseResult = fetchResults.get(option.getHit().getShard().getShardId().id()); + assertSame(searchPhaseResult.getSearchShardTarget(), option.getHit().getShard()); } } } @@ -193,8 +202,9 @@ private static AtomicArray generateQueryResults(int nShards, int searchHitsSize, boolean useConstantScore) { AtomicArray queryResults = new AtomicArray<>(nShards); for (int shardIndex = 0; shardIndex < nShards; shardIndex++) { - QuerySearchResult querySearchResult = new QuerySearchResult(shardIndex, - new SearchShardTarget("", new Index("", ""), shardIndex, null)); + CCSInfo ccsInfo = randomBoolean() ? null : new CCSInfo("", randomBoolean()); + SearchShardTarget searchShardTarget = new SearchShardTarget("", new ShardId("", "", shardIndex), ccsInfo, OriginalIndices.NONE); + QuerySearchResult querySearchResult = new QuerySearchResult(shardIndex, searchShardTarget); final TopDocs topDocs; float maxScore = 0; if (searchHitsSize == 0) { @@ -237,7 +247,7 @@ private static AtomicArray generateQueryResults(int nShards, return queryResults; } - private int getTotalQueryHits(AtomicArray results) { + private static int getTotalQueryHits(AtomicArray results) { int resultCount = 0; for (SearchPhaseResult shardResult : results.asList()) { TopDocs topDocs = shardResult.queryResult().topDocs().topDocs; @@ -247,7 +257,7 @@ private int getTotalQueryHits(AtomicArray results) { return resultCount; } - private Suggest reducedSuggest(AtomicArray results) { + private static Suggest reducedSuggest(AtomicArray results) { Map>> groupedSuggestion = new HashMap<>(); for (SearchPhaseResult entry : results.asList()) { for (Suggest.Suggestion suggestion : entry.queryResult().suggest()) { @@ -260,11 +270,12 @@ private Suggest reducedSuggest(AtomicArray results) { .collect(Collectors.toList())); } - private AtomicArray generateFetchResults(int nShards, ScoreDoc[] mergedSearchDocs, Suggest mergedSuggest) { + private static AtomicArray generateFetchResults(int nShards, ScoreDoc[] mergedSearchDocs, Suggest mergedSuggest) { AtomicArray fetchResults = new AtomicArray<>(nShards); for (int shardIndex = 0; shardIndex < nShards; shardIndex++) { float maxScore = -1F; - SearchShardTarget shardTarget = new SearchShardTarget("", new Index("", ""), shardIndex, null); + CCSInfo ccsInfo = randomBoolean() ? null : new CCSInfo("", randomBoolean()); + SearchShardTarget shardTarget = new SearchShardTarget("", new ShardId("", "", shardIndex), ccsInfo, OriginalIndices.NONE); FetchSearchResult fetchSearchResult = new FetchSearchResult(shardIndex, shardTarget); List searchHits = new ArrayList<>(); for (ScoreDoc scoreDoc : mergedSearchDocs) { diff --git a/server/src/test/java/org/elasticsearch/search/SearchRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java similarity index 81% rename from server/src/test/java/org/elasticsearch/search/SearchRequestTests.java rename to server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java index ced279a1babd6..ce130f388015d 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java @@ -17,27 +17,39 @@ * under the License. */ -package org.elasticsearch.search; +package org.elasticsearch.action.search; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.ArrayUtils; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.AbstractSearchTestCase; +import org.elasticsearch.search.Scroll; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.rescore.QueryRescorerBuilder; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.VersionUtils; import java.io.IOException; import java.util.ArrayList; +import java.util.Base64; import java.util.List; import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; public class SearchRequestTests extends AbstractSearchTestCase { + @Override + protected SearchRequest createSearchRequest() throws IOException { + SearchRequest searchRequest = super.createSearchRequest(); + //clusterAlias does not have public getter/setter hence we randomize it only in this test specifically. + searchRequest.setClusterAlias(randomAlphaOfLengthBetween(5, 10)); + return searchRequest; + } + public void testSerialization() throws Exception { SearchRequest searchRequest = createSearchRequest(); SearchRequest deserializedRequest = copyWriteable(searchRequest, namedWriteableRegistry, SearchRequest::new); @@ -46,6 +58,28 @@ public void testSerialization() throws Exception { assertNotSame(deserializedRequest, searchRequest); } + public void testClusterAliasSerialization() throws IOException { + SearchRequest searchRequest = createSearchRequest(); + Version version = VersionUtils.randomVersion(random()); + SearchRequest deserializedRequest = copyWriteable(searchRequest, namedWriteableRegistry, SearchRequest::new, version); + //TODO update version after backport + if (version.before(Version.V_7_0_0)) { + assertNull(deserializedRequest.getClusterAlias()); + } else { + assertEquals(searchRequest.getClusterAlias(), deserializedRequest.getClusterAlias()); + } + } + + //TODO rename and update version after backport + public void testReadFromPre7_0_0() throws IOException { + String msg = "AAEBBWluZGV4AAAAAQACAAAA/////w8AAAAAAAAA/////w8AAAAAAAACAAAAAAABAAMCBAUBAAKABACAAQIAAA=="; + try (StreamInput in = StreamInput.wrap(Base64.getDecoder().decode(msg))) { + SearchRequest searchRequest = new SearchRequest(in); + assertArrayEquals(new String[]{"index"}, searchRequest.indices()); + assertNull(searchRequest.getClusterAlias()); + } + } + public void testIllegalArguments() { SearchRequest searchRequest = new SearchRequest(); assertNotNull(searchRequest.indices()); @@ -140,11 +174,11 @@ public void testCopyConstructor() throws IOException { } public void testEqualsAndHashcode() throws IOException { - checkEqualsAndHashCode(createSearchRequest(), SearchRequestTests::copyRequest, this::mutate); + checkEqualsAndHashCode(createSearchRequest(), SearchRequest::new, this::mutate); } private SearchRequest mutate(SearchRequest searchRequest) { - SearchRequest mutation = copyRequest(searchRequest); + SearchRequest mutation = new SearchRequest(searchRequest); List mutators = new ArrayList<>(); mutators.add(() -> mutation.indices(ArrayUtils.concat(searchRequest.indices(), new String[] { randomAlphaOfLength(10) }))); mutators.add(() -> mutation.indicesOptions(randomValueOtherThan(searchRequest.indicesOptions(), @@ -158,11 +192,9 @@ private SearchRequest mutate(SearchRequest searchRequest) { mutators.add(() -> mutation.searchType(randomValueOtherThan(searchRequest.searchType(), () -> randomFrom(SearchType.DFS_QUERY_THEN_FETCH, SearchType.QUERY_THEN_FETCH)))); mutators.add(() -> mutation.source(randomValueOtherThan(searchRequest.source(), this::createSearchSourceBuilder))); + mutators.add(() -> mutation.setClusterAlias(randomValueOtherThan(searchRequest.getClusterAlias(), + () -> randomAlphaOfLengthBetween(3, 10)))); randomFrom(mutators).run(); return mutation; } - - private static SearchRequest copyRequest(SearchRequest searchRequest) { - return new SearchRequest(searchRequest); - } } diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java index ecd2dc44a702e..65a37d0c8686c 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java @@ -182,7 +182,7 @@ public void testFromXContentWithFailures() throws IOException { int numFailures = randomIntBetween(1, 5); ShardSearchFailure[] failures = new ShardSearchFailure[numFailures]; for (int i = 0; i < failures.length; i++) { - failures[i] = ShardSearchFailureTests.createTestItem(IndexMetaData.INDEX_UUID_NA_VALUE); + failures[i] = ShardSearchFailureTests.createTestItem(IndexMetaData.INDEX_UUID_NA_VALUE, false); } SearchResponse response = createTestItem(failures); XContentType xcontentType = randomFrom(XContentType.values()); diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchShardIteratorTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchShardIteratorTests.java new file mode 100644 index 0000000000000..a3ee739b81181 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/search/SearchShardIteratorTests.java @@ -0,0 +1,78 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.search; + +import org.elasticsearch.action.OriginalIndices; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.search.CCSInfo; +import org.elasticsearch.search.SearchShardTarget; +import org.elasticsearch.test.ESTestCase; + +import java.util.Collections; + +public class SearchShardIteratorTests extends ESTestCase { + + public void testShardId() { + ShardId shardId = new ShardId(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLength(10), randomInt()); + SearchShardIterator searchShardIterator = new SearchShardIterator(null, shardId, Collections.emptyList(), OriginalIndices.NONE); + assertSame(shardId, searchShardIterator.shardId()); + } + + public void testGetOriginalIndices() { + ShardId shardId = new ShardId(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLength(10), randomInt()); + OriginalIndices originalIndices = new OriginalIndices(new String[]{randomAlphaOfLengthBetween(3, 10)}, + IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean())); + SearchShardIterator searchShardIterator = new SearchShardIterator(null, shardId, Collections.emptyList(), originalIndices); + assertSame(originalIndices, searchShardIterator.getOriginalIndices()); + } + + public void testCCSInfo() { + CCSInfo ccsInfo = randomBoolean() ? null : new CCSInfo(randomAlphaOfLengthBetween(5, 10), randomBoolean()); + ShardId shardId = new ShardId(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLength(10), randomInt()); + SearchShardIterator searchShardIterator = new SearchShardIterator(ccsInfo, shardId, Collections.emptyList(), OriginalIndices.NONE); + assertSame(ccsInfo, searchShardIterator.getCCSInfo()); + if (ccsInfo == null) { + assertNull(searchShardIterator.getCCSInfo()); + } else { + assertEquals(ccsInfo.getConnectionAlias(), searchShardIterator.getCCSInfo().getConnectionAlias()); + } + } + + public void testNewSearchShardTarget() { + CCSInfo ccsInfo = randomBoolean() ? null : new CCSInfo(randomAlphaOfLengthBetween(5, 10), randomBoolean()); + ShardId shardId = new ShardId(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLength(10), randomInt()); + OriginalIndices originalIndices = new OriginalIndices(new String[]{randomAlphaOfLengthBetween(3, 10)}, + IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean())); + SearchShardIterator searchShardIterator = new SearchShardIterator(ccsInfo, shardId, Collections.emptyList(), originalIndices); + String nodeId = randomAlphaOfLengthBetween(3, 10); + SearchShardTarget searchShardTarget = searchShardIterator.newSearchShardTarget(nodeId); + if (ccsInfo == null) { + assertNull(searchShardTarget.getConnectionAlias()); + assertNull(searchShardTarget.getHitIndexPrefix()); + } else { + assertEquals(ccsInfo.getConnectionAlias(), searchShardTarget.getConnectionAlias()); + assertEquals(ccsInfo.getHitIndexPrefix(), searchShardTarget.getHitIndexPrefix()); + } + assertSame(shardId, searchShardTarget.getShardId()); + assertEquals(nodeId, searchShardTarget.getNodeId()); + assertSame(originalIndices, searchShardTarget.getOriginalIndices()); + } +} diff --git a/server/src/test/java/org/elasticsearch/action/search/ShardSearchFailureTests.java b/server/src/test/java/org/elasticsearch/action/search/ShardSearchFailureTests.java index f62f874c9e2c9..c18ee4808672e 100644 --- a/server/src/test/java/org/elasticsearch/action/search/ShardSearchFailureTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/ShardSearchFailureTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.search; +import org.elasticsearch.Version; import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.ParsingException; @@ -28,6 +29,7 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.search.CCSInfo; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; @@ -39,16 +41,17 @@ public class ShardSearchFailureTests extends ESTestCase { - public static ShardSearchFailure createTestItem(String indexUuid) { + public static ShardSearchFailure createTestItem(String indexUuid, boolean transportSerialization) { String randomMessage = randomAlphaOfLengthBetween(3, 20); Exception ex = new ParsingException(0, 0, randomMessage , new IllegalArgumentException("some bad argument")); SearchShardTarget searchShardTarget = null; if (randomBoolean()) { String nodeId = randomAlphaOfLengthBetween(5, 10); String indexName = randomAlphaOfLengthBetween(5, 10); - String clusterAlias = randomBoolean() ? randomAlphaOfLengthBetween(5, 10) : null; + CCSInfo ccsInfo = randomBoolean() ? null : + new CCSInfo(randomAlphaOfLengthBetween(5, 10), transportSerialization && randomBoolean()); searchShardTarget = new SearchShardTarget(nodeId, - new ShardId(new Index(indexName, indexUuid), randomInt()), clusterAlias, OriginalIndices.NONE); + new ShardId(new Index(indexName, indexUuid), randomInt()), ccsInfo, OriginalIndices.NONE); } return new ShardSearchFailure(ex, searchShardTarget); } @@ -67,7 +70,7 @@ public void testFromXContentWithRandomFields() throws IOException { } private void doFromXContentTestWithRandomFields(boolean addRandomFields) throws IOException { - ShardSearchFailure response = createTestItem(IndexMetaData.INDEX_UUID_NA_VALUE); + ShardSearchFailure response = createTestItem(IndexMetaData.INDEX_UUID_NA_VALUE, false); XContentType xContentType = randomFrom(XContentType.values()); boolean humanReadable = randomBoolean(); BytesReference originalBytes = toShuffledXContent(response, xContentType, ToXContent.EMPTY_PARAMS, humanReadable); @@ -120,7 +123,8 @@ public void testToXContent() throws IOException { public void testToXContentWithClusterAlias() throws IOException { ShardSearchFailure failure = new ShardSearchFailure(new ParsingException(0, 0, "some message", null), - new SearchShardTarget("nodeId", new ShardId(new Index("indexName", "indexUuid"), 123), "cluster1", OriginalIndices.NONE)); + new SearchShardTarget("nodeId", new ShardId(new Index("indexName", "indexUuid"), 123), + new CCSInfo("cluster1", randomBoolean()), OriginalIndices.NONE)); BytesReference xContent = toXContent(failure, XContentType.JSON, randomBoolean()); assertEquals( "{\"shard\":123," @@ -137,13 +141,35 @@ public void testToXContentWithClusterAlias() throws IOException { } public void testSerialization() throws IOException { - ShardSearchFailure testItem = createTestItem(randomAlphaOfLength(12)); + ShardSearchFailure testItem = createTestItem(randomAlphaOfLength(12), true); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_7_0_0, Version.CURRENT); ShardSearchFailure deserializedInstance = copyStreamable(testItem, writableRegistry(), - ShardSearchFailure::new, VersionUtils.randomVersion(random())); + ShardSearchFailure::new, version); assertEquals(testItem.index(), deserializedInstance.index()); assertEquals(testItem.shard(), deserializedInstance.shard()); assertEquals(testItem.shardId(), deserializedInstance.shardId()); assertEquals(testItem.reason(), deserializedInstance.reason()); assertEquals(testItem.status(), deserializedInstance.status()); } + + //TODO rename and update version after backport + public void testSerializationPre7_0_0() throws IOException { + ShardSearchFailure testItem = createTestItem(randomAlphaOfLength(12), true); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, VersionUtils.getPreviousVersion(Version.V_7_0_0)); + ShardSearchFailure deserializedInstance = copyStreamable(testItem, writableRegistry(), ShardSearchFailure::new, version); + assertEquals(testItem.index(), deserializedInstance.index()); + if (testItem.shard() == null) { + assertNull(deserializedInstance.shard()); + } else { + //equality check on target doesn't work as CCSInfo is partially read (missing boolean localReduction flag set to false) + assertEquals(testItem.shard().getNodeId(), deserializedInstance.shard().getNodeId()); + assertEquals(testItem.shard().getShardId(), deserializedInstance.shard().getShardId()); + assertEquals(testItem.shard().getHitIndexPrefix(), deserializedInstance.shard().getHitIndexPrefix()); + assertEquals(testItem.shard().getHitIndexPrefix(), deserializedInstance.shard().getConnectionAlias()); + } + assertEquals(testItem.shardId(), deserializedInstance.shardId()); + assertEquals(testItem.reason(), deserializedInstance.reason()); + assertEquals(testItem.status(), deserializedInstance.status()); + } + } diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java index 3e4747a4db757..079fc6c4da95c 100644 --- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java @@ -36,6 +36,7 @@ import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.search.CCSInfo; import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.transport.MockTransportService; @@ -85,17 +86,18 @@ public void testMergeShardsIterators() { OriginalIndices remoteIndices = new OriginalIndices(new String[]{"remote_alias", "remote_index_2"}, IndicesOptions.strictExpandOpen()); List remoteShardIterators = new ArrayList<>(); + CCSInfo remoteCCSInfo = new CCSInfo("remote", false); { ShardId remoteShardId = new ShardId("remote_index", "remote_index_uuid", 2); ShardRouting remoteShardRouting = TestShardRouting.newShardRouting(remoteShardId, "remote_node", true, STARTED); - SearchShardIterator remoteShardIterator = new SearchShardIterator("remote", remoteShardId, + SearchShardIterator remoteShardIterator = new SearchShardIterator(remoteCCSInfo, remoteShardId, Collections.singletonList(remoteShardRouting), remoteIndices); remoteShardIterators.add(remoteShardIterator); } { ShardId remoteShardId2 = new ShardId("remote_index_2", "remote_index_2_uuid", 3); ShardRouting remoteShardRouting2 = TestShardRouting.newShardRouting(remoteShardId2, "remote_node", true, STARTED); - SearchShardIterator remoteShardIterator2 = new SearchShardIterator("remote", remoteShardId2, + SearchShardIterator remoteShardIterator2 = new SearchShardIterator(remoteCCSInfo, remoteShardId2, Collections.singletonList(remoteShardRouting2), remoteIndices); remoteShardIterators.add(remoteShardIterator2); } @@ -104,13 +106,14 @@ public void testMergeShardsIterators() { { ShardId remoteShardId3 = new ShardId("remote_index_3", "remote_index_3_uuid", 4); ShardRouting remoteShardRouting3 = TestShardRouting.newShardRouting(remoteShardId3, "remote_node", true, STARTED); - SearchShardIterator remoteShardIterator3 = new SearchShardIterator("remote", remoteShardId3, + SearchShardIterator remoteShardIterator3 = new SearchShardIterator(remoteCCSInfo, remoteShardId3, Collections.singletonList(remoteShardRouting3), remoteIndices2); remoteShardIterators.add(remoteShardIterator3); } + String localClusterAlias = randomBoolean() ? null : "local"; GroupShardsIterator searchShardIterators = TransportSearchAction.mergeShardsIterators(localShardsIterator, - localIndices, remoteShardIterators); + localIndices, localClusterAlias, remoteShardIterators); assertEquals(searchShardIterators.size(), 5); int i = 0; @@ -120,31 +123,54 @@ public void testMergeShardsIterators() { assertEquals("local_index", searchShardIterator.shardId().getIndexName()); assertEquals(0, searchShardIterator.shardId().getId()); assertSame(localIndices, searchShardIterator.getOriginalIndices()); + assertLocalClusterAlias(searchShardIterator, localClusterAlias); break; case 1: assertEquals("local_index_2", searchShardIterator.shardId().getIndexName()); assertEquals(1, searchShardIterator.shardId().getId()); assertSame(localIndices, searchShardIterator.getOriginalIndices()); + assertLocalClusterAlias(searchShardIterator, localClusterAlias); break; case 2: assertEquals("remote_index", searchShardIterator.shardId().getIndexName()); assertEquals(2, searchShardIterator.shardId().getId()); assertSame(remoteIndices, searchShardIterator.getOriginalIndices()); + assertRemoteClusterAlias(searchShardIterator, remoteCCSInfo); break; case 3: assertEquals("remote_index_2", searchShardIterator.shardId().getIndexName()); assertEquals(3, searchShardIterator.shardId().getId()); assertSame(remoteIndices, searchShardIterator.getOriginalIndices()); + assertRemoteClusterAlias(searchShardIterator, remoteCCSInfo); break; case 4: assertEquals("remote_index_3", searchShardIterator.shardId().getIndexName()); assertEquals(4, searchShardIterator.shardId().getId()); assertSame(remoteIndices2, searchShardIterator.getOriginalIndices()); + assertRemoteClusterAlias(searchShardIterator, remoteCCSInfo); break; } } } + private static void assertRemoteClusterAlias(SearchShardIterator searchShardIterator, CCSInfo ccsInfo) { + assertSame(ccsInfo, searchShardIterator.getCCSInfo()); + assertEquals("remote", searchShardIterator.getConnectionAlias()); + assertEquals("remote", searchShardIterator.getCCSInfo().getConnectionAlias()); + assertEquals("remote", searchShardIterator.getCCSInfo().getHitIndexPrefix()); + } + + private static void assertLocalClusterAlias(SearchShardIterator searchShardIterator, String localClusterAlias) { + if (localClusterAlias == null) { + assertNull(searchShardIterator.getCCSInfo()); + assertNull(searchShardIterator.getConnectionAlias()); + } else { + assertEquals(localClusterAlias, searchShardIterator.getCCSInfo().getHitIndexPrefix()); + assertNull(searchShardIterator.getCCSInfo().getConnectionAlias()); + assertNull(searchShardIterator.getConnectionAlias()); + } + } + public void testProcessRemoteShards() { try (TransportService transportService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { @@ -197,7 +223,8 @@ public void testProcessRemoteShards() { assertArrayEquals(new String[]{"some_alias_for_foo", "some_other_foo_alias"}, iterator.getOriginalIndices().indices()); assertTrue(iterator.shardId().getId() == 0 || iterator.shardId().getId() == 1); - assertEquals("test_cluster_1", iterator.getClusterAlias()); + assertEquals("test_cluster_1", iterator.getConnectionAlias()); + assertEquals(new CCSInfo("test_cluster_1", false), iterator.getCCSInfo()); assertEquals("foo", iterator.shardId().getIndexName()); ShardRouting shardRouting = iterator.nextOrNull(); assertNotNull(shardRouting); @@ -209,7 +236,8 @@ public void testProcessRemoteShards() { } else if (iterator.shardId().getIndexName().endsWith("bar")) { assertArrayEquals(new String[]{"bar"}, iterator.getOriginalIndices().indices()); assertEquals(0, iterator.shardId().getId()); - assertEquals("test_cluster_1", iterator.getClusterAlias()); + assertEquals("test_cluster_1", iterator.getConnectionAlias()); + assertEquals(new CCSInfo("test_cluster_1", false), iterator.getCCSInfo()); assertEquals("bar", iterator.shardId().getIndexName()); ShardRouting shardRouting = iterator.nextOrNull(); assertNotNull(shardRouting); @@ -222,7 +250,8 @@ public void testProcessRemoteShards() { assertArrayEquals(new String[]{"some_alias_for_xyz"}, iterator.getOriginalIndices().indices()); assertEquals(0, iterator.shardId().getId()); assertEquals("xyz", iterator.shardId().getIndexName()); - assertEquals("test_cluster_2", iterator.getClusterAlias()); + assertEquals("test_cluster_2", iterator.getConnectionAlias()); + assertEquals(new CCSInfo("test_cluster_2", false), iterator.getCCSInfo()); ShardRouting shardRouting = iterator.nextOrNull(); assertNotNull(shardRouting); assertEquals(shardRouting.getIndexName(), "xyz"); diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java index 49d7450096bb5..a84101da94cb9 100644 --- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.search.CCSInfo; import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.test.ESTestCase; @@ -36,16 +37,17 @@ public void testParseScrollId() throws IOException { DiscoveryNode node2 = new DiscoveryNode("node_2", buildNewFakeTransportAddress(), Version.CURRENT); DiscoveryNode node3 = new DiscoveryNode("node_3", buildNewFakeTransportAddress(), Version.CURRENT); SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult1 = new SearchAsyncActionTests.TestSearchPhaseResult(1, node1); - testSearchPhaseResult1.setSearchShardTarget(new SearchShardTarget("node_1", new ShardId("idx", "uuid1", 2), "cluster_x", null)); + SearchShardTarget target = new SearchShardTarget("node_1", new ShardId("idx", "uuid1", 2), new CCSInfo("cluster_x", false), null); + testSearchPhaseResult1.setSearchShardTarget(target); SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult2 = new SearchAsyncActionTests.TestSearchPhaseResult(12, node2); - testSearchPhaseResult2.setSearchShardTarget(new SearchShardTarget("node_2", new ShardId("idy", "uuid2", 42), "cluster_y", null)); + SearchShardTarget target2 = new SearchShardTarget("node_2", new ShardId("idy", "uuid2", 42), new CCSInfo("cluster_y", false), null); + testSearchPhaseResult2.setSearchShardTarget(target2); SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult3 = new SearchAsyncActionTests.TestSearchPhaseResult(42, node3); testSearchPhaseResult3.setSearchShardTarget(new SearchShardTarget("node_3", new ShardId("idy", "uuid2", 43), null, null)); array.setOnce(0, testSearchPhaseResult1); array.setOnce(1, testSearchPhaseResult2); array.setOnce(2, testSearchPhaseResult3); - String scrollId = TransportSearchHelper.buildScrollId(array); ParsedScrollId parseScrollId = TransportSearchHelper.parseScrollId(scrollId); assertEquals(3, parseScrollId.getContext().length); diff --git a/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java b/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java index 8d547c617e55b..d0f7ddcadc3c0 100644 --- a/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java +++ b/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.Rewriteable; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.search.CCSInfo; import org.elasticsearch.search.Scroll; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.internal.AliasFilter; @@ -43,11 +44,11 @@ import java.io.IOException; import java.util.Collections; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.hasToString; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.startsWith; public class SearchSlowLogTests extends ESSingleNodeTestCase { @@ -154,7 +155,7 @@ public Rewriteable getRewriteable() { } @Override - public String getClusterAlias() { + public CCSInfo getCCSInfo() { return null; } }; diff --git a/server/src/test/java/org/elasticsearch/rest/action/RestActionsTests.java b/server/src/test/java/org/elasticsearch/rest/action/RestActionsTests.java index d44b2b70a05ef..668962ccb68ec 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/RestActionsTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/RestActionsTests.java @@ -36,6 +36,7 @@ import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.search.CCSInfo; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.test.ESTestCase; @@ -198,8 +199,9 @@ private static ShardSearchFailure createShardFailureParsingException(String node } private static SearchShardTarget createSearchShardTarget(String nodeId, int shardId, String index, String clusterAlias) { + CCSInfo ccsInfo = clusterAlias == null ? null : new CCSInfo(clusterAlias, randomBoolean()); return new SearchShardTarget(nodeId, - new ShardId(new Index(index, IndexMetaData.INDEX_UUID_NA_VALUE), shardId), clusterAlias, OriginalIndices.NONE); + new ShardId(new Index(index, IndexMetaData.INDEX_UUID_NA_VALUE), shardId), ccsInfo, OriginalIndices.NONE); } @Override diff --git a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java index 4b86ff668c0de..189929171a5d1 100644 --- a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java +++ b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java @@ -26,6 +26,7 @@ import org.apache.lucene.search.Sort; import org.apache.lucene.store.Directory; import org.elasticsearch.Version; +import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; @@ -110,10 +111,12 @@ public void testPreProcess() throws Exception { try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir); IndexReader reader = w.getReader(); - Engine.Searcher searcher = new Engine.Searcher("test", new IndexSearcher(reader), reader::close)) { + Engine.Searcher searcher = new Engine.Searcher("test", new IndexSearcher(reader), reader)) { - DefaultSearchContext context1 = new DefaultSearchContext(1L, shardSearchRequest, null, searcher, null, indexService, - indexShard, bigArrays, null, timeout, null, null, Version.CURRENT); + SearchShardTarget target = new SearchShardTarget("node", shardId, null, OriginalIndices.NONE); + + DefaultSearchContext context1 = new DefaultSearchContext(1L, shardSearchRequest, target, searcher, null, indexService, + indexShard, bigArrays, null, timeout, null, Version.CURRENT); context1.from(300); // resultWindow greater than maxResultWindow and scrollContext is null @@ -153,8 +156,8 @@ public void testPreProcess() throws Exception { + "] index level setting.")); // rescore is null but sliceBuilder is not null - DefaultSearchContext context2 = new DefaultSearchContext(2L, shardSearchRequest, null, searcher, - null, indexService, indexShard, bigArrays, null, timeout, null, null, Version.CURRENT); + DefaultSearchContext context2 = new DefaultSearchContext(2L, shardSearchRequest, target, searcher, + null, indexService, indexShard, bigArrays, null, timeout, null, Version.CURRENT); SliceBuilder sliceBuilder = mock(SliceBuilder.class); int numSlices = maxSlicesPerScroll + randomIntBetween(1, 100); @@ -170,8 +173,8 @@ public void testPreProcess() throws Exception { when(shardSearchRequest.getAliasFilter()).thenReturn(AliasFilter.EMPTY); when(shardSearchRequest.indexBoost()).thenReturn(AbstractQueryBuilder.DEFAULT_BOOST); - DefaultSearchContext context3 = new DefaultSearchContext(3L, shardSearchRequest, null, searcher, null, - indexService, indexShard, bigArrays, null, timeout, null, null, Version.CURRENT); + DefaultSearchContext context3 = new DefaultSearchContext(3L, shardSearchRequest, target, searcher, null, + indexService, indexShard, bigArrays, null, timeout, null, Version.CURRENT); ParsedQuery parsedQuery = ParsedQuery.parsedMatchAllQuery(); context3.sliceBuilder(null).parsedQuery(parsedQuery).preProcess(false); assertEquals(context3.query(), context3.buildFilteredQuery(parsedQuery.query())); diff --git a/server/src/test/java/org/elasticsearch/search/SearchHitTests.java b/server/src/test/java/org/elasticsearch/search/SearchHitTests.java index 3ad39404afefb..50cb587df103d 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchHitTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchHitTests.java @@ -21,13 +21,13 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.TotalHits; +import org.elasticsearch.Version; import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.document.DocumentField; -import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.common.io.stream.InputStreamStreamInput; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -42,10 +42,11 @@ import org.elasticsearch.search.fetch.subphase.highlight.HighlightFieldTests; import org.elasticsearch.test.AbstractStreamableTestCase; import org.elasticsearch.test.RandomObjects; +import org.elasticsearch.test.VersionUtils; import java.io.IOException; -import java.io.InputStream; import java.util.ArrayList; +import java.util.Base64; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -127,9 +128,9 @@ public static SearchHit createTestItem(XContentType xContentType, boolean withOp } if (transportSerialization && randomBoolean()) { String index = randomAlphaOfLengthBetween(5, 10); - String clusterAlias = randomBoolean() ? null : randomAlphaOfLengthBetween(5, 10); + CCSInfo ccsInfo = randomBoolean() ? null : new CCSInfo(randomAlphaOfLengthBetween(5, 10), randomBoolean()); hit.shard(new SearchShardTarget(randomAlphaOfLengthBetween(5, 10), - new ShardId(new Index(index, randomAlphaOfLengthBetween(5, 10)), randomInt()), clusterAlias, OriginalIndices.NONE)); + new ShardId(new Index(index, randomAlphaOfLengthBetween(5, 10)), randomInt()), ccsInfo, OriginalIndices.NONE)); } return hit; } @@ -213,8 +214,9 @@ public void testToXContent() throws IOException { } public void testSerializeShardTarget() throws Exception { - String clusterAlias = randomBoolean() ? null : "cluster_alias"; - SearchShardTarget target = new SearchShardTarget("_node_id", new Index("_index", "_na_"), 0, clusterAlias); + CCSInfo ccsInfo = randomBoolean() ? null : new CCSInfo("cluster_alias", randomBoolean()); + SearchShardTarget target = new SearchShardTarget("_node_id", new ShardId(new Index("_index", "_na_"), 0), + ccsInfo, OriginalIndices.NONE); Map innerHits = new HashMap<>(); SearchHit innerHit1 = new SearchHit(0, "_id", new Text("_type"), null); @@ -240,30 +242,57 @@ public void testSerializeShardTarget() throws Exception { SearchHits hits = new SearchHits(new SearchHit[]{hit1, hit2}, new TotalHits(2, TotalHits.Relation.EQUAL_TO), 1f); - - BytesStreamOutput output = new BytesStreamOutput(); - hits.writeTo(output); - InputStream input = output.bytes().streamInput(); - SearchHits results = SearchHits.readSearchHits(new InputStreamStreamInput(input)); - assertThat(results.getAt(0).getShard(), equalTo(target)); + Version version = VersionUtils.randomVersion(random()); + SearchHits results = copyStreamable(hits, getNamedWriteableRegistry(), SearchHits::new, version); + SearchShardTarget deserializedTarget = results.getAt(0).getShard(); + //TODO update version after backport + if (version.before(Version.V_7_0_0) && ccsInfo != null) { + //localReduction flag is not serialized as part of CCSInfo + target = new SearchShardTarget(target.getNodeId(), target.getShardId(), + new CCSInfo(ccsInfo.getHitIndexPrefix(), false), target.getOriginalIndices()); + } + assertThat(deserializedTarget, equalTo(target)); assertThat(results.getAt(0).getInnerHits().get("1").getAt(0).getShard(), notNullValue()); assertThat(results.getAt(0).getInnerHits().get("1").getAt(0).getInnerHits().get("1").getAt(0).getShard(), notNullValue()); assertThat(results.getAt(0).getInnerHits().get("1").getAt(1).getShard(), notNullValue()); assertThat(results.getAt(0).getInnerHits().get("2").getAt(0).getShard(), notNullValue()); for (SearchHit hit : results) { - assertEquals(clusterAlias, hit.getClusterAlias()); + assertEquals(ccsInfo == null ? null : ccsInfo.getHitIndexPrefix(), hit.getClusterAlias()); if (hit.getInnerHits() != null) { for (SearchHits innerhits : hit.getInnerHits().values()) { for (SearchHit innerHit : innerhits) { - assertEquals(clusterAlias, innerHit.getClusterAlias()); + assertEquals(ccsInfo == null ? null : ccsInfo.getHitIndexPrefix(), innerHit.getClusterAlias()); } } } } - assertThat(results.getAt(1).getShard(), equalTo(target)); } + //TODO rename and update version after backport + public void testReadFromPre7_0_0() throws IOException { + String msg = "/4AAAAAAAAJpZP////8A//////////8AAAAAAAAAAQEAAAAEbm9kZQVpbmRleAR1dWlkAAEOcmVtb3RlX2NsdXN0ZXIBA2tl" + + "eQEAf8AAAAAAAAAAAAAAAAAAAAAAAAAA"; + SearchHit hit = new SearchHit(); + try (StreamInput in = StreamInput.wrap(Base64.getDecoder().decode(msg))) { + //TODO randomize the version after backport, conflicts with SearchSortValues wire changes otherwise + in.setVersion(Version.V_6_6_0); + hit.readFrom(in); + } + assertEquals("id", hit.getId()); + assertEquals(0, hit.docId()); + SearchShardTarget shardTarget = hit.getShard(); + assertEquals("node", shardTarget.getNodeId()); + assertEquals(new ShardId(new Index("index", "uuid"), 0), shardTarget.getShardId()); + assertEquals("remote_cluster", shardTarget.getHitIndexPrefix()); + assertEquals("remote_cluster", shardTarget.getConnectionAlias()); + Map innerHits = hit.getInnerHits(); + assertEquals(1, innerHits.size()); + SearchHits searchHits = innerHits.get("key"); + assertEquals(0, searchHits.getHits().length); + assertEquals(0, searchHits.getTotalHits().value); + } + public void testNullSource() { SearchHit searchHit = new SearchHit(0, "_id", new Text("_type"), null); diff --git a/server/src/test/java/org/elasticsearch/search/SearchHitsTests.java b/server/src/test/java/org/elasticsearch/search/SearchHitsTests.java index 8623c2021bffd..2239d659f45e4 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchHitsTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchHitsTests.java @@ -232,9 +232,9 @@ public void testFromXContentWithShards() throws IOException { for (SearchHit hit : hits) { String index = randomAlphaOfLengthBetween(5, 10); - String clusterAlias = randomBoolean() ? null : randomAlphaOfLengthBetween(5, 10); + CCSInfo ccsInfo = randomBoolean() ? null : new CCSInfo(randomAlphaOfLengthBetween(5, 10), randomBoolean()); final SearchShardTarget shardTarget = new SearchShardTarget(randomAlphaOfLengthBetween(5, 10), - new ShardId(new Index(index, randomAlphaOfLengthBetween(5, 10)), randomInt()), clusterAlias, OriginalIndices.NONE); + new ShardId(new Index(index, randomAlphaOfLengthBetween(5, 10)), randomInt()), ccsInfo, OriginalIndices.NONE); if (withExplanation) { hit.explanation(SearchHitTests.createExplanation(randomIntBetween(0, 5))); } @@ -291,6 +291,15 @@ public void testSerializationPre6_6_0() throws IOException { Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, VersionUtils.getPreviousVersion(Version.V_6_6_0)); SearchHits original = createTestItem(randomFrom(XContentType.values()), false, true, TotalHits.Relation.EQUAL_TO); SearchHits deserialized = copyInstance(original, version); + for (SearchHit originalHit : original.getHits()) { + SearchShardTarget shard = originalHit.getShard(); + if (shard != null && shard.getHitIndexPrefix() != null) { + //CCSInfo is not fully read, localReduction is always set to false when reading from previous versions + SearchShardTarget newTarget = new SearchShardTarget(shard.getNodeId(), shard.getShardId(), + new CCSInfo(shard.getHitIndexPrefix(), false), shard.getOriginalIndices()); + originalHit.shard(newTarget); + } + } assertArrayEquals(original.getHits(), deserialized.getHits()); assertEquals(original.getMaxScore(), deserialized.getMaxScore(), 0F); if (original.getTotalHits() == null) { diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java index 30598311ad574..4116c05abc940 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java @@ -23,12 +23,14 @@ import org.apache.lucene.store.AlreadyClosedException; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.search.ClearScrollRequest; import org.elasticsearch.action.search.SearchPhaseExecutionException; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchTask; import org.elasticsearch.action.search.SearchType; -import org.elasticsearch.action.search.ClearScrollRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.WriteRequest; @@ -71,14 +73,16 @@ import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.internal.ShardSearchLocalRequest; +import org.elasticsearch.search.internal.ShardSearchTransportRequest; import org.elasticsearch.search.suggest.SuggestBuilder; import org.elasticsearch.test.ESSingleNodeTestCase; import java.io.IOException; import java.util.Collection; import java.util.Collections; -import java.util.List; import java.util.LinkedList; +import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; @@ -114,7 +118,6 @@ public static class CustomScriptPlugin extends MockScriptPlugin { static final String DUMMY_SCRIPT = "dummyScript"; - @Override protected Map, Object>> pluginScripts() { return Collections.singletonMap(DUMMY_SCRIPT, vars -> "dummy"); @@ -151,7 +154,6 @@ public void onQueryPhase(SearchContext context, long tookInNanos) { } }); } - } @Override @@ -637,4 +639,34 @@ public void testCreateReduceContext() { reduceContext.consumeBucketsAndMaybeBreak(MultiBucketConsumerService.DEFAULT_MAX_BUCKETS + 1); } } + + public void testCreateSearchContext() throws IOException { + String index = randomAlphaOfLengthBetween(5, 10).toLowerCase(Locale.ROOT); + IndexService indexService = createIndex(index); + final SearchService service = getInstanceFromNode(SearchService.class); + ShardId shardId = new ShardId(indexService.index(), 0); + long nowInMillis = System.currentTimeMillis(); + CCSInfo ccsInfo = randomBoolean() ? null : new CCSInfo(randomAlphaOfLengthBetween(3, 10), randomBoolean()); + SearchRequest searchRequest = new SearchRequest(); + searchRequest.allowPartialSearchResults(randomBoolean()); + ShardSearchTransportRequest request = new ShardSearchTransportRequest(OriginalIndices.NONE, searchRequest, shardId, + indexService.numberOfShards(), AliasFilter.EMPTY, 1f, nowInMillis, ccsInfo, Strings.EMPTY_ARRAY); + DefaultSearchContext searchContext = service.createSearchContext(request, new TimeValue(System.currentTimeMillis())); + SearchShardTarget searchShardTarget = searchContext.shardTarget(); + QueryShardContext queryShardContext = searchContext.getQueryShardContext(); + String expectedIndexName = ccsInfo == null ? index : ccsInfo.getHitIndexPrefix() + ":" + index; + assertEquals(expectedIndexName, queryShardContext.getFullyQualifiedIndex().getName()); + assertEquals(expectedIndexName, searchShardTarget.getFullyQualifiedIndexName()); + if (ccsInfo == null) { + assertNull(searchShardTarget.getHitIndexPrefix()); + assertNull(searchShardTarget.getConnectionAlias()); + } else { + assertEquals(ccsInfo.getHitIndexPrefix(), searchShardTarget.getHitIndexPrefix()); + assertEquals(ccsInfo.getConnectionAlias(), searchShardTarget.getConnectionAlias()); + } + assertEquals(shardId, searchShardTarget.getShardId()); + assertSame(searchShardTarget, searchContext.dfsResult().getSearchShardTarget()); + assertSame(searchShardTarget, searchContext.queryResult().getSearchShardTarget()); + assertSame(searchShardTarget, searchContext.fetchResult().getSearchShardTarget()); + } } diff --git a/server/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java b/server/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java index 21a4f099f5a32..f9ee772fccf3d 100644 --- a/server/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java +++ b/server/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java @@ -28,8 +28,6 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; -import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.DeprecationHandler; @@ -43,8 +41,11 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.InvalidAliasNameException; import org.elasticsearch.search.AbstractSearchTestCase; +import org.elasticsearch.search.CCSInfo; +import org.elasticsearch.test.VersionUtils; import java.io.IOException; +import java.util.Base64; import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; import static org.elasticsearch.index.query.QueryBuilders.termQuery; @@ -59,28 +60,49 @@ public class ShardSearchTransportRequestTests extends AbstractSearchTestCase { public void testSerialization() throws Exception { ShardSearchTransportRequest shardSearchTransportRequest = createShardSearchTransportRequest(); - try (BytesStreamOutput output = new BytesStreamOutput()) { - shardSearchTransportRequest.writeTo(output); - try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { - ShardSearchTransportRequest deserializedRequest = new ShardSearchTransportRequest(in); - assertEquals(deserializedRequest.scroll(), shardSearchTransportRequest.scroll()); - assertEquals(deserializedRequest.getAliasFilter(), shardSearchTransportRequest.getAliasFilter()); - assertArrayEquals(deserializedRequest.indices(), shardSearchTransportRequest.indices()); - assertArrayEquals(deserializedRequest.types(), shardSearchTransportRequest.types()); - assertEquals(deserializedRequest.indicesOptions(), shardSearchTransportRequest.indicesOptions()); - assertEquals(deserializedRequest.isProfile(), shardSearchTransportRequest.isProfile()); - assertEquals(deserializedRequest.nowInMillis(), shardSearchTransportRequest.nowInMillis()); - assertEquals(deserializedRequest.source(), shardSearchTransportRequest.source()); - assertEquals(deserializedRequest.searchType(), shardSearchTransportRequest.searchType()); - assertEquals(deserializedRequest.shardId(), shardSearchTransportRequest.shardId()); - assertEquals(deserializedRequest.numberOfShards(), shardSearchTransportRequest.numberOfShards()); - assertEquals(deserializedRequest.indexRoutings(), shardSearchTransportRequest.indexRoutings()); - assertEquals(deserializedRequest.preference(), shardSearchTransportRequest.preference()); - assertEquals(deserializedRequest.cacheKey(), shardSearchTransportRequest.cacheKey()); - assertNotSame(deserializedRequest, shardSearchTransportRequest); - assertEquals(deserializedRequest.getAliasFilter(), shardSearchTransportRequest.getAliasFilter()); - assertEquals(deserializedRequest.indexBoost(), shardSearchTransportRequest.indexBoost(), 0.0f); - } + ShardSearchTransportRequest deserializedRequest = + copyWriteable(shardSearchTransportRequest, namedWriteableRegistry, ShardSearchTransportRequest::new); + assertEquals(deserializedRequest.scroll(), shardSearchTransportRequest.scroll()); + assertEquals(deserializedRequest.getAliasFilter(), shardSearchTransportRequest.getAliasFilter()); + assertArrayEquals(deserializedRequest.indices(), shardSearchTransportRequest.indices()); + assertArrayEquals(deserializedRequest.types(), shardSearchTransportRequest.types()); + assertEquals(deserializedRequest.indicesOptions(), shardSearchTransportRequest.indicesOptions()); + assertEquals(deserializedRequest.isProfile(), shardSearchTransportRequest.isProfile()); + assertEquals(deserializedRequest.nowInMillis(), shardSearchTransportRequest.nowInMillis()); + assertEquals(deserializedRequest.source(), shardSearchTransportRequest.source()); + assertEquals(deserializedRequest.searchType(), shardSearchTransportRequest.searchType()); + assertEquals(deserializedRequest.shardId(), shardSearchTransportRequest.shardId()); + assertEquals(deserializedRequest.numberOfShards(), shardSearchTransportRequest.numberOfShards()); + assertArrayEquals(deserializedRequest.indexRoutings(), shardSearchTransportRequest.indexRoutings()); + assertEquals(deserializedRequest.preference(), shardSearchTransportRequest.preference()); + assertEquals(deserializedRequest.cacheKey(), shardSearchTransportRequest.cacheKey()); + assertNotSame(deserializedRequest, shardSearchTransportRequest); + assertEquals(deserializedRequest.getAliasFilter(), shardSearchTransportRequest.getAliasFilter()); + assertEquals(deserializedRequest.indexBoost(), shardSearchTransportRequest.indexBoost(), 0.0f); + assertEquals(deserializedRequest.getCCSInfo(), shardSearchTransportRequest.getCCSInfo()); + } + + //TODO rename and update version after backport + public void testSerializationPre7_0_0() throws IOException { + Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, VersionUtils.getPreviousVersion(Version.V_7_0_0)); + ShardSearchTransportRequest shardSearchTransportRequest = createShardSearchTransportRequest(); + ShardSearchTransportRequest deserializedRequest = + copyWriteable(shardSearchTransportRequest, namedWriteableRegistry, ShardSearchTransportRequest::new, version); + assertEquals(new CCSInfo(shardSearchTransportRequest.getCCSInfo().getHitIndexPrefix(), false), deserializedRequest.getCCSInfo()); + } + + //TODO rename and update version after backport + public void testReadFromPre7_0_0() throws IOException { + String msg = "AAVpbmRleAR1dWlkAAEBAAEAAgAAAP////8PAAAAAAAAAP////8PAAAAAAAAAgAAAAAAAQAAAD+" + + "AAAAAAgEGcmVtb3RlAAAAAQVpbmRleAMCBAUBAAAAAAAAAAAAAAAA"; + try (StreamInput in = StreamInput.wrap(Base64.getDecoder().decode(msg))) { + in.setVersion(VersionUtils.randomVersionBetween(random(), Version.V_6_4_0, VersionUtils.getPreviousVersion(Version.V_7_0_0))); + ShardSearchTransportRequest request = new ShardSearchTransportRequest(in); + assertArrayEquals(new String[]{"index"}, request.indices()); + assertEquals(new ShardId("index", "uuid", 0), request.shardId()); + assertEquals(1, request.numberOfShards()); + assertEquals("remote", request.getCCSInfo().getHitIndexPrefix()); + assertEquals("remote", request.getCCSInfo().getConnectionAlias()); } } @@ -97,7 +119,7 @@ private ShardSearchTransportRequest createShardSearchTransportRequest() throws I final String[] routings = generateRandomStringArray(5, 10, false, true); return new ShardSearchTransportRequest(new OriginalIndices(searchRequest), searchRequest, shardId, randomIntBetween(1, 100), filteringAliases, randomBoolean() ? 1.0f : randomFloat(), - Math.abs(randomLong()), null, routings); + Math.abs(randomLong()), new CCSInfo(randomAlphaOfLengthBetween(3, 10), randomBoolean()), routings); } public void testFilteringAliases() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java b/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java index 30ed0cb5ab5b5..8d996a7a1b87c 100644 --- a/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java @@ -58,6 +58,7 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.Rewriteable; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.search.CCSInfo; import org.elasticsearch.search.Scroll; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.internal.AliasFilter; @@ -198,7 +199,7 @@ public BytesReference cacheKey() throws IOException { } @Override - public String getClusterAlias() { + public CCSInfo getCCSInfo() { return null; } From 019e3ca2a0642ea7349a636d5a17d27590131466 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Thu, 27 Dec 2018 15:51:46 +0100 Subject: [PATCH 2/9] Remove CCSInfo class Simplify changes by making connection lookup aware of the search request cluster alias, so there is no need to distinguish between index prefix and connection alias, which means no serialization changes required besides adding support for cluster alias to the search request. --- .../search/AbstractSearchAsyncAction.java | 3 +- .../search/CanMatchPreFilterSearchPhase.java | 2 +- .../action/search/DfsQueryPhase.java | 2 +- .../action/search/FetchSearchPhase.java | 4 +- .../SearchDfsQueryThenFetchAsyncAction.java | 2 +- .../SearchQueryThenFetchAsyncAction.java | 2 +- .../search/SearchScrollAsyncAction.java | 6 +- ...SearchScrollQueryThenFetchAsyncAction.java | 4 +- .../action/search/SearchShardIterator.java | 32 ++-- .../action/search/ShardSearchFailure.java | 3 +- .../action/search/TransportSearchAction.java | 14 +- .../action/search/TransportSearchHelper.java | 2 +- .../org/elasticsearch/search/CCSInfo.java | 137 ------------------ .../search/DefaultSearchContext.java | 2 +- .../org/elasticsearch/search/SearchHit.java | 4 +- .../elasticsearch/search/SearchService.java | 2 +- .../search/SearchShardTarget.java | 38 ++--- .../internal/ShardSearchLocalRequest.java | 15 +- .../search/internal/ShardSearchRequest.java | 10 +- .../internal/ShardSearchTransportRequest.java | 10 +- .../elasticsearch/ExceptionsHelperTests.java | 4 +- .../AbstractSearchAsyncActionTests.java | 13 +- .../search/SearchPhaseControllerTests.java | 10 +- .../search/SearchShardIteratorTests.java | 26 +--- .../search/ShardSearchFailureTests.java | 34 +---- .../search/TransportSearchActionTests.java | 45 ++---- .../search/TransportSearchHelperTests.java | 5 +- .../index/SearchSlowLogTests.java | 3 +- .../rest/action/RestActionsTests.java | 4 +- .../elasticsearch/search/SearchHitTests.java | 44 +----- .../elasticsearch/search/SearchHitsTests.java | 13 +- .../search/SearchServiceTests.java | 14 +- .../ShardSearchTransportRequestTests.java | 35 +---- .../search/slice/SliceBuilderTests.java | 3 +- 34 files changed, 113 insertions(+), 434 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/search/CCSInfo.java diff --git a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java index fccbf9f6b2354..2da609e906c9c 100644 --- a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java @@ -318,6 +318,7 @@ public final void onFailure(Exception e) { listener.onFailure(e); } + @Override public final ShardSearchTransportRequest buildShardSearchRequest(SearchShardIterator shardIt) { AliasFilter filter = aliasFilter.get(shardIt.shardId().getIndex().getUUID()); assert filter != null; @@ -326,7 +327,7 @@ public final ShardSearchTransportRequest buildShardSearchRequest(SearchShardIter final String[] routings = indexRoutings.getOrDefault(indexName, Collections.emptySet()) .toArray(new String[0]); return new ShardSearchTransportRequest(shardIt.getOriginalIndices(), request, shardIt.shardId(), getNumShards(), - filter, indexBoost, timeProvider.getAbsoluteStartMillis(), shardIt.getCCSInfo(), routings); + filter, indexBoost, timeProvider.getAbsoluteStartMillis(), shardIt.getClusterAlias(), routings); } /** diff --git a/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java index cc3552eb09378..bea5a975b2840 100644 --- a/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java @@ -66,7 +66,7 @@ final class CanMatchPreFilterSearchPhase extends AbstractSearchAsyncAction listener) { - getSearchTransport().sendCanMatch(getConnection(shardIt.getConnectionAlias(), shard.currentNodeId()), + getSearchTransport().sendCanMatch(getConnection(shardIt.getClusterAlias(), shard.currentNodeId()), buildShardSearchRequest(shardIt), getTask(), listener); } diff --git a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java index 17a1d399d7a2c..1d8d702520e4c 100644 --- a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java @@ -71,7 +71,7 @@ public void run() throws IOException { () -> context.executeNextPhase(this, nextPhaseFactory.apply(queryResult)), context); for (final DfsSearchResult dfsResult : resultList) { final SearchShardTarget searchShardTarget = dfsResult.getSearchShardTarget(); - Transport.Connection connection = context.getConnection(searchShardTarget.getConnectionAlias(), searchShardTarget.getNodeId()); + Transport.Connection connection = context.getConnection(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId()); QuerySearchRequest querySearchRequest = new QuerySearchRequest(searchShardTarget.getOriginalIndices(), dfsResult.getRequestId(), dfs); final int shardIndex = dfsResult.getShardIndex(); diff --git a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java index 1bf993b6f4c39..a86d638abe342 100644 --- a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java @@ -137,7 +137,7 @@ private void innerRun() throws IOException { counter.countDown(); } else { SearchShardTarget searchShardTarget = queryResult.getSearchShardTarget(); - Transport.Connection connection = context.getConnection(searchShardTarget.getConnectionAlias(), + Transport.Connection connection = context.getConnection(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId()); ShardFetchSearchRequest fetchSearchRequest = createFetchRequest(queryResult.queryResult().getRequestId(), i, entry, lastEmittedDocPerShard, searchShardTarget.getOriginalIndices()); @@ -190,7 +190,7 @@ private void releaseIrrelevantSearchContext(QuerySearchResult queryResult) { if (context.getRequest().scroll() == null && queryResult.hasSearchContext()) { try { SearchShardTarget searchShardTarget = queryResult.getSearchShardTarget(); - Transport.Connection connection = context.getConnection(searchShardTarget.getConnectionAlias(), + Transport.Connection connection = context.getConnection(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId()); context.sendReleaseSearchContext(queryResult.getRequestId(), connection, searchShardTarget.getOriginalIndices()); } catch (Exception e) { diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java index e06522a367b1c..0782fbb310b65 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java @@ -53,7 +53,7 @@ final class SearchDfsQueryThenFetchAsyncAction extends AbstractSearchAsyncAction @Override protected void executePhaseOnShard(final SearchShardIterator shardIt, final ShardRouting shard, final SearchActionListener listener) { - getSearchTransport().sendExecuteDfs(getConnection(shardIt.getConnectionAlias(), shard.currentNodeId()), + getSearchTransport().sendExecuteDfs(getConnection(shardIt.getClusterAlias(), shard.currentNodeId()), buildShardSearchRequest(shardIt) , getTask(), listener); } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java index 04f1f281cdad7..bbd84011de00b 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java @@ -51,7 +51,7 @@ final class SearchQueryThenFetchAsyncAction extends AbstractSearchAsyncAction listener) { - getSearchTransport().sendExecuteQuery(getConnection(shardIt.getConnectionAlias(), shard.currentNodeId()), + getSearchTransport().sendExecuteQuery(getConnection(shardIt.getClusterAlias(), shard.currentNodeId()), buildShardSearchRequest(shardIt), getTask(), listener); } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java index f83393d759db2..81d7e66d19edb 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java @@ -27,7 +27,6 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.common.util.concurrent.CountDown; -import org.elasticsearch.search.CCSInfo; import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.internal.InternalScrollSearchRequest; @@ -174,11 +173,8 @@ protected void setSearchShardTarget(T response) { // re-create the search target and add the cluster alias if there is any, // we need this down the road for subseq. phases SearchShardTarget searchShardTarget = response.getSearchShardTarget(); - //We can assume that we are executing as part of a CCS request with a single coordination step. - //Scroll is not supported when performing multiple coordination steps in each cluster. - CCSInfo ccsInfo = new CCSInfo(target.getClusterAlias(), false); response.setSearchShardTarget(new SearchShardTarget(searchShardTarget.getNodeId(), searchShardTarget.getShardId(), - ccsInfo, null)); + target.getClusterAlias(), null)); } } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java index 326c8e494d8d5..df18296de2a4a 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java @@ -89,9 +89,9 @@ public void run() { ShardFetchRequest shardFetchRequest = new ShardFetchRequest(querySearchResult.getRequestId(), docIds, lastEmittedDoc); SearchShardTarget searchShardTarget = querySearchResult.getSearchShardTarget(); - DiscoveryNode node = clusterNodeLookup.apply(searchShardTarget.getConnectionAlias(), searchShardTarget.getNodeId()); + DiscoveryNode node = clusterNodeLookup.apply(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId()); assert node != null : "target node is null in secondary phase"; - Transport.Connection connection = getConnection(searchShardTarget.getConnectionAlias(), node); + Transport.Connection connection = getConnection(searchShardTarget.getClusterAlias(), node); searchTransportService.sendExecuteFetchScroll(connection, shardFetchRequest, task, new SearchActionListener(querySearchResult.getSearchShardTarget(), index) { @Override diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java b/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java index e51f2f86dd18f..3e1d2ddbbdd97 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java @@ -24,7 +24,6 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.Nullable; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.search.CCSInfo; import org.elasticsearch.search.SearchShardTarget; import java.util.List; @@ -32,29 +31,28 @@ /** * Extension of {@link PlainShardIterator} used in the search api, which also holds the {@link OriginalIndices} * of the search request (useful especially with cross-cluster search, as each cluster has its own set of original indices) as well as - * information about the cross-cluster search execution. + * the cluster alias. * @see OriginalIndices - * @see CCSInfo */ public final class SearchShardIterator extends PlainShardIterator { private final OriginalIndices originalIndices; - private final CCSInfo ccsInfo; + private final String clusterAlias; private boolean skip = false; /** * Creates a {@link PlainShardIterator} instance that iterates over a subset of the given shards * this the a given shardId. * - * @param ccsInfo information about cross-cluster search execution, null if not applicable + * @param clusterAlias the alias of the cluster where the shard is located * @param shardId shard id of the group * @param shards shards to iterate * @param originalIndices the indices that the search request originally related to (before any rewriting happened) */ - public SearchShardIterator(@Nullable CCSInfo ccsInfo, ShardId shardId, List shards, OriginalIndices originalIndices) { + public SearchShardIterator(@Nullable String clusterAlias, ShardId shardId, List shards, OriginalIndices originalIndices) { super(shardId, shards); this.originalIndices = originalIndices; - this.ccsInfo = ccsInfo; + this.clusterAlias = clusterAlias; } /** @@ -65,23 +63,11 @@ public OriginalIndices getOriginalIndices() { } /** - * Returns the cluster alias needed to lookup the connection when sending shard level requests in the context of this iterator. - * null indicates that the shard is local, meaning that either we are not executing a cross-cluster search request, - * or we are but each cluster performs its own reduction. + * Returns the alias of the cluster where the shard is located */ @Nullable - public String getConnectionAlias() { - return ccsInfo == null ? null : ccsInfo.getConnectionAlias(); - } - - /** - * Returns info about cross-cluster search execution if applicable. - * null means that we are not executing a cross-cluster search request. - * @see CCSInfo - */ - @Nullable - public CCSInfo getCCSInfo() { - return ccsInfo; + public String getClusterAlias() { + return clusterAlias; } /** @@ -89,7 +75,7 @@ public CCSInfo getCCSInfo() { * @see SearchShardTarget */ SearchShardTarget newSearchShardTarget(String nodeId) { - return new SearchShardTarget(nodeId, shardId(), ccsInfo, originalIndices); + return new SearchShardTarget(nodeId, shardId(), clusterAlias, originalIndices); } /** diff --git a/server/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java b/server/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java index 6546a9ddb2a76..7927d4b0f6278 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java +++ b/server/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java @@ -32,7 +32,6 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.search.CCSInfo; import org.elasticsearch.search.SearchException; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.transport.RemoteClusterAware; @@ -171,7 +170,7 @@ public static ShardSearchFailure fromXContent(XContentParser parser) throws IOEx SearchShardTarget searchShardTarget = null; if (nodeId != null) { searchShardTarget = new SearchShardTarget(nodeId, new ShardId(new Index(indexName, IndexMetaData.INDEX_UUID_NA_VALUE), shardId), - CCSInfo.fromClusterAlias(clusterAlias), OriginalIndices.NONE); + clusterAlias, OriginalIndices.NONE); } return new ShardSearchFailure(exception, searchShardTarget); } diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index 06e6fe4d6989e..a74a33473cd1e 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -42,7 +42,6 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.query.Rewriteable; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.search.CCSInfo; import org.elasticsearch.search.SearchService; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.internal.AliasFilter; @@ -264,7 +263,7 @@ static BiFunction processRemoteShards(Map connectionLookup = (clusterName, nodeId) -> { - final DiscoveryNode discoveryNode = clusterName == null ? nodes.get(nodeId) : remoteConnections.apply(clusterName, nodeId); + final DiscoveryNode discoveryNode; + if (clusterName == null || searchRequest.getClusterAlias() != null) { + assert searchRequest.getClusterAlias() == null || searchRequest.getClusterAlias().equals(clusterName); + discoveryNode = nodes.get(nodeId); + } else { + discoveryNode = remoteConnections.apply(clusterName, nodeId); + } if (discoveryNode == null) { throw new IllegalStateException("no node found for id: " + nodeId); } @@ -363,10 +368,9 @@ static GroupShardsIterator mergeShardsIterators(GroupShards OriginalIndices localIndices, @Nullable String localClusterAlias, List remoteShardIterators) { - CCSInfo localCCSInfo = localClusterAlias == null ? null : new CCSInfo(localClusterAlias, true); List shards = new ArrayList<>(remoteShardIterators); for (ShardIterator shardIterator : localShardsIterator) { - shards.add(new SearchShardIterator(localCCSInfo, shardIterator.shardId(), shardIterator.getShardRoutings(), localIndices)); + shards.add(new SearchShardIterator(localClusterAlias, shardIterator.shardId(), shardIterator.getShardRoutings(), localIndices)); } return new GroupShardsIterator<>(shards); } diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java index d3c14f18ca9f5..3a4e01c09c9af 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java @@ -43,7 +43,7 @@ static String buildScrollId(AtomicArray searchPhase for (SearchPhaseResult searchPhaseResult : searchPhaseResults.asList()) { out.writeLong(searchPhaseResult.getRequestId()); SearchShardTarget searchShardTarget = searchPhaseResult.getSearchShardTarget(); - String hitIndexPrefix = searchShardTarget.getHitIndexPrefix(); + String hitIndexPrefix = searchShardTarget.getClusterAlias(); if (hitIndexPrefix != null) { out.writeString(RemoteClusterAware.buildRemoteIndexName(hitIndexPrefix, searchShardTarget.getNodeId())); } else { diff --git a/server/src/main/java/org/elasticsearch/search/CCSInfo.java b/server/src/main/java/org/elasticsearch/search/CCSInfo.java deleted file mode 100644 index 631b7dba5ce5f..0000000000000 --- a/server/src/main/java/org/elasticsearch/search/CCSInfo.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search; - -import org.elasticsearch.Version; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; - -import java.io.IOException; -import java.util.Objects; - -/** - * Encloses information about a cross-cluster search request: the cluster alias that will be used to prefix the index name returned - * with the {@link SearchHit}, and whether the sub-request is part of a locally reduced cross-cluster search request. - * In case a cross-cluster search request fans out to all the (local and remote) shards in one go, the cluster alias has to be used - * as both the index prefix and the identifier for {@link org.elasticsearch.transport.Transport.Connection} lookup when sending each - * shard level request. On the other hand, when reducing results locally in each remote cluster, the cluster alias has to be used only - * as the index prefix, all the targeted shards in each sub-search request are local. - */ -public final class CCSInfo implements Writeable { - private final String clusterAlias; - private final boolean localReduction; - - public CCSInfo(String clusterAlias, boolean localReduction) { - this.clusterAlias = Objects.requireNonNull(clusterAlias); - this.localReduction = localReduction; - } - - private CCSInfo(StreamInput in) throws IOException { - this.clusterAlias = in.readString(); - this.localReduction = in.readBoolean(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeString(clusterAlias); - out.writeBoolean(localReduction); - } - - /** - * Returns the prefix to be used for the index names of each search result, to indicate which cluster returned it. - */ - public String getHitIndexPrefix() { - return clusterAlias; - } - - /** - * Returns the cluster alias needed to lookup the connection when sending shard level requests. null indicates that the - * shards are local, hence we are executing a cross-cluster search request as part of which each cluster performs its own reduction. - */ - @Nullable - public String getConnectionAlias() { - return localReduction ? null : clusterAlias; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - CCSInfo ccsInfo = (CCSInfo) o; - return localReduction == ccsInfo.localReduction && - Objects.equals(clusterAlias, ccsInfo.clusterAlias); - } - - @Override - public int hashCode() { - return Objects.hash(clusterAlias, localReduction); - } - - /** - * Reads an optional {@link CCSInfo} instance from the provided {@link StreamInput}. - * Handles backwards compatibility with previous versions that supported only the cluster alias as an optional string. In such - * cases the localReduction flag will be set to false as CCS local reductions were not supported anyways. - */ - @Nullable - public static CCSInfo read(StreamInput in) throws IOException { - //TODO update version once backported - if (in.getVersion().onOrAfter(Version.V_7_0_0)) { - return in.readOptionalWriteable(CCSInfo::new); - } - if (in.readBoolean()) { - return new CCSInfo(in.readString(), false); - } - return null; - } - - /** - * Writes an optional {@link CCSInfo} instance to the provided {@link StreamOutput}. - * Handles backwards compatibility with previous versions that supported only the cluster alias as an optional string. In such - * cases the localReduction flag will be set to false as CCS local reductions were not supported anyways. - */ - public static void write(@Nullable CCSInfo ccsInfo, StreamOutput out) throws IOException { - //TODO update version once backported - if (out.getVersion().onOrAfter(Version.V_7_0_0)) { - out.writeOptionalWriteable(ccsInfo); - } else { - if (ccsInfo == null) { - out.writeBoolean(false); - } else { - out.writeOptionalString(ccsInfo.clusterAlias); - } - } - } - - /** - * Returns a new {@link CCSInfo} instance or null depending on the provided cluster alias. - * This method is used as a central place to build a {@link CCSInfo} when the cluster alias may be null, hence null - * should be returned, and when the localReduction flag is not available (as it's not serialized at REST) hence false will be assumed. - */ - @Nullable - public static CCSInfo fromClusterAlias(@Nullable String clusterAlias) { - return clusterAlias == null ? null : new CCSInfo(clusterAlias, false); - } -} diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 4d4d3323d086d..05e90214e15d1 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -179,7 +179,7 @@ final class DefaultSearchContext extends SearchContext { this.timeout = timeout; this.minNodeVersion = minNodeVersion; queryShardContext = indexService.newQueryShardContext(request.shardId().id(), searcher.getIndexReader(), request::nowInMillis, - shardTarget.getHitIndexPrefix()); + shardTarget.getClusterAlias()); queryShardContext.setTypes(request.types()); queryBoost = request.indexBoost(); } diff --git a/server/src/main/java/org/elasticsearch/search/SearchHit.java b/server/src/main/java/org/elasticsearch/search/SearchHit.java index 3a87505cb1965..7fd68852ce284 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchHit.java +++ b/server/src/main/java/org/elasticsearch/search/SearchHit.java @@ -354,7 +354,7 @@ public void shard(SearchShardTarget target) { this.shard = target; if (target != null) { this.index = target.getIndex(); - this.clusterAlias = target.getHitIndexPrefix(); + this.clusterAlias = target.getClusterAlias(); } } @@ -578,7 +578,7 @@ public static SearchHit createFromMap(Map values) { String nodeId = get(Fields._NODE, values, null); if (shardId != null && nodeId != null) { assert shardId.getIndexName().equals(index); - searchHit.shard(new SearchShardTarget(nodeId, shardId, CCSInfo.fromClusterAlias(clusterAlias), OriginalIndices.NONE)); + searchHit.shard(new SearchShardTarget(nodeId, shardId, clusterAlias, OriginalIndices.NONE)); } else { //these fields get set anyways when setting the shard target, //but we set them explicitly when we don't have enough info to rebuild the shard target diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index 2c6f9a0d594ed..500e70a65b486 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -678,7 +678,7 @@ private DefaultSearchContext createSearchContext(ShardSearchRequest request, Tim IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex()); IndexShard indexShard = indexService.getShard(request.shardId().getId()); SearchShardTarget shardTarget = new SearchShardTarget(clusterService.localNode().getId(), - indexShard.shardId(), request.getCCSInfo(), OriginalIndices.NONE); + indexShard.shardId(), request.getClusterAlias(), OriginalIndices.NONE); Engine.Searcher engineSearcher = indexShard.acquireSearcher(source); final DefaultSearchContext searchContext = new DefaultSearchContext(idGenerator.incrementAndGet(), request, shardTarget, diff --git a/server/src/main/java/org/elasticsearch/search/SearchShardTarget.java b/server/src/main/java/org/elasticsearch/search/SearchShardTarget.java index 2301a29b5c153..42f3b67e358e4 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchShardTarget.java +++ b/server/src/main/java/org/elasticsearch/search/SearchShardTarget.java @@ -42,7 +42,7 @@ public final class SearchShardTarget implements Writeable, Comparablenull means that no prefix is needed, meaning that we are not executing a cross-cluster search request. - */ - @Nullable - public String getHitIndexPrefix() { - return ccsInfo == null ? null : ccsInfo.getHitIndexPrefix(); - } - - /** - * Returns the cluster alias needed to lookup the connection when sending shard level requests in the context of this shard target. - * null indicates that the shard is local, meaning that either we are not executing a cross-cluster search request, - * or we are but each cluster performs its own reduction. - */ @Nullable - public String getConnectionAlias() { - return ccsInfo == null ? null : ccsInfo.getConnectionAlias(); + public String getClusterAlias() { + return clusterAlias; } /** * Returns the fully qualified index name, including the index prefix that indicates which cluster results come from. */ public String getFullyQualifiedIndexName() { - return RemoteClusterAware.buildRemoteIndexName(getHitIndexPrefix(), getIndex()); + return RemoteClusterAware.buildRemoteIndexName(clusterAlias, getIndex()); } @Override @@ -132,7 +118,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeText(nodeId); } shardId.writeTo(out); - CCSInfo.write(ccsInfo, out); + out.writeOptionalString(clusterAlias); } @Override @@ -146,18 +132,18 @@ public boolean equals(Object o) { SearchShardTarget that = (SearchShardTarget) o; return Objects.equals(nodeId, that.nodeId) && Objects.equals(shardId, that.shardId) && - Objects.equals(ccsInfo, that.ccsInfo); + Objects.equals(clusterAlias, that.clusterAlias); } @Override public int hashCode() { - return Objects.hash(nodeId, shardId, ccsInfo); + return Objects.hash(nodeId, shardId, clusterAlias); } @Override public String toString() { String shardToString = "[" + RemoteClusterAware.buildRemoteIndexName( - getHitIndexPrefix(), shardId.getIndexName()) + "][" + shardId.getId() + "]"; + clusterAlias, shardId.getIndexName()) + "][" + shardId.getId() + "]"; if (nodeId == null) { return "[_na_]" + shardToString; } diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java index 83c101c5aceed..0921681124e33 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java @@ -32,7 +32,6 @@ import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.Rewriteable; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.search.CCSInfo; import org.elasticsearch.search.Scroll; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -59,7 +58,7 @@ */ public class ShardSearchLocalRequest implements ShardSearchRequest { - private CCSInfo ccsInfo; + private String clusterAlias; private ShardId shardId; private int numberOfShards; private SearchType searchType; @@ -79,7 +78,7 @@ public class ShardSearchLocalRequest implements ShardSearchRequest { } ShardSearchLocalRequest(SearchRequest searchRequest, ShardId shardId, int numberOfShards, AliasFilter aliasFilter, float indexBoost, - long nowInMillis, @Nullable CCSInfo ccsInfo, String[] indexRoutings) { + long nowInMillis, @Nullable String clusterAlias, String[] indexRoutings) { this(shardId, numberOfShards, searchRequest.searchType(), searchRequest.source(), searchRequest.types(), searchRequest.requestCache(), aliasFilter, indexBoost, searchRequest.allowPartialSearchResults(), indexRoutings, searchRequest.preference()); @@ -88,7 +87,7 @@ public class ShardSearchLocalRequest implements ShardSearchRequest { assert searchRequest.allowPartialSearchResults() != null; this.scroll = searchRequest.scroll(); this.nowInMillis = nowInMillis; - this.ccsInfo = ccsInfo; + this.clusterAlias = clusterAlias; } public ShardSearchLocalRequest(ShardId shardId, String[] types, long nowInMillis, AliasFilter aliasFilter) { @@ -216,7 +215,7 @@ protected void innerReadFrom(StreamInput in) throws IOException { indexBoost = in.readFloat(); nowInMillis = in.readVLong(); requestCache = in.readOptionalBoolean(); - ccsInfo = CCSInfo.read(in); + clusterAlias = in.readOptionalString(); if (in.getVersion().onOrAfter(Version.V_6_3_0)) { allowPartialSearchResults = in.readOptionalBoolean(); } @@ -244,7 +243,7 @@ protected void innerWriteTo(StreamOutput out, boolean asKey) throws IOException out.writeVLong(nowInMillis); } out.writeOptionalBoolean(requestCache); - CCSInfo.write(ccsInfo, out); + out.writeOptionalString(clusterAlias); if (out.getVersion().onOrAfter(Version.V_6_3_0)) { out.writeOptionalBoolean(allowPartialSearchResults); } @@ -266,8 +265,8 @@ public BytesReference cacheKey() throws IOException { } @Override - public CCSInfo getCCSInfo() { - return ccsInfo; + public String getClusterAlias() { + return clusterAlias; } @Override diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java index d7e6123f95b5c..2de73af1a1b5c 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java @@ -24,6 +24,7 @@ import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.CheckedFunction; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.index.Index; @@ -33,7 +34,6 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.AliasFilterParsingException; import org.elasticsearch.indices.InvalidAliasNameException; -import org.elasticsearch.search.CCSInfo; import org.elasticsearch.search.Scroll; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -151,12 +151,8 @@ static QueryBuilder parseAliasFilter(CheckedFunctionnull otherwise - * @see CCSInfo - */ - CCSInfo getCCSInfo(); + @Nullable + String getClusterAlias(); Rewriteable getRewriteable(); diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java index 3436f5ed96a15..59d1c2e089e02 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java @@ -25,12 +25,12 @@ import org.elasticsearch.action.search.SearchTask; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.index.query.Rewriteable; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.search.CCSInfo; import org.elasticsearch.search.Scroll; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.tasks.Task; @@ -54,9 +54,9 @@ public class ShardSearchTransportRequest extends TransportRequest implements Sha public ShardSearchTransportRequest(OriginalIndices originalIndices, SearchRequest searchRequest, ShardId shardId, int numberOfShards, AliasFilter aliasFilter, float indexBoost, long nowInMillis, - CCSInfo ccsInfo, String[] indexRoutings) { + @Nullable String clusterAlias, String[] indexRoutings) { this.shardSearchLocalRequest = new ShardSearchLocalRequest(searchRequest, shardId, numberOfShards, aliasFilter, indexBoost, - nowInMillis, ccsInfo, indexRoutings); + nowInMillis, clusterAlias, indexRoutings); this.originalIndices = originalIndices; } @@ -201,8 +201,8 @@ public String getDescription() { } @Override - public CCSInfo getCCSInfo() { - return shardSearchLocalRequest.getCCSInfo(); + public String getClusterAlias() { + return shardSearchLocalRequest.getClusterAlias(); } @Override diff --git a/server/src/test/java/org/elasticsearch/ExceptionsHelperTests.java b/server/src/test/java/org/elasticsearch/ExceptionsHelperTests.java index d9f8996ccbe79..1d2a4ca6d5f75 100644 --- a/server/src/test/java/org/elasticsearch/ExceptionsHelperTests.java +++ b/server/src/test/java/org/elasticsearch/ExceptionsHelperTests.java @@ -30,7 +30,6 @@ import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.search.CCSInfo; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.transport.RemoteClusterAware; @@ -137,9 +136,8 @@ private static ShardSearchFailure createShardFailureParsingException(String erro } private static SearchShardTarget createSearchShardTarget(String nodeId, int shardId, String index, String clusterAlias) { - CCSInfo ccsInfo = clusterAlias == null ? null : new CCSInfo(clusterAlias, randomBoolean()); return new SearchShardTarget(nodeId, - new ShardId(new Index(index, IndexMetaData.INDEX_UUID_NA_VALUE), shardId), ccsInfo, OriginalIndices.NONE); + new ShardId(new Index(index, IndexMetaData.INDEX_UUID_NA_VALUE), shardId), clusterAlias, OriginalIndices.NONE); } public void testGroupByNullTarget() { diff --git a/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java index 84e21f7be464c..70f70268a0a03 100644 --- a/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java @@ -27,7 +27,6 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.search.CCSInfo; import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.search.internal.ShardSearchTransportRequest; @@ -117,8 +116,8 @@ private void runTestTook(final boolean controlled) { public void testBuildShardSearchTransportRequest() { final AtomicLong expected = new AtomicLong(); AbstractSearchAsyncAction action = createAction(false, expected); - CCSInfo ccsInfo = randomBoolean() ? null : new CCSInfo(randomAlphaOfLengthBetween(5, 10), randomBoolean()); - SearchShardIterator iterator = new SearchShardIterator(ccsInfo, new ShardId(new Index("name", "foo"), 1), + String clusterAlias = randomBoolean() ? null : randomAlphaOfLengthBetween(5, 10); + SearchShardIterator iterator = new SearchShardIterator(clusterAlias, new ShardId(new Index("name", "foo"), 1), Collections.emptyList(), new OriginalIndices(new String[] {"name", "name1"}, IndicesOptions.strictExpand())); ShardSearchTransportRequest shardSearchTransportRequest = action.buildShardSearchRequest(iterator); assertEquals(IndicesOptions.strictExpand(), shardSearchTransportRequest.indicesOptions()); @@ -128,12 +127,6 @@ public void testBuildShardSearchTransportRequest() { assertArrayEquals(new String[] {"name", "name1"}, shardSearchTransportRequest.indices()); assertArrayEquals(new String[] {"bar", "baz"}, shardSearchTransportRequest.indexRoutings()); assertEquals("_shards:1,3", shardSearchTransportRequest.preference()); - assertSame(ccsInfo, shardSearchTransportRequest.getCCSInfo()); - if (ccsInfo == null) { - assertNull(shardSearchTransportRequest.getCCSInfo()); - } else { - assertEquals(ccsInfo.getHitIndexPrefix(), shardSearchTransportRequest.getCCSInfo().getHitIndexPrefix()); - assertEquals(ccsInfo.getConnectionAlias(), shardSearchTransportRequest.getCCSInfo().getConnectionAlias()); - } + assertEquals(clusterAlias, shardSearchTransportRequest.getClusterAlias()); } } diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java index 3170e17f618ac..585108fef8a32 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java @@ -37,7 +37,6 @@ import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.search.CCSInfo; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; @@ -202,8 +201,9 @@ private static AtomicArray generateQueryResults(int nShards, int searchHitsSize, boolean useConstantScore) { AtomicArray queryResults = new AtomicArray<>(nShards); for (int shardIndex = 0; shardIndex < nShards; shardIndex++) { - CCSInfo ccsInfo = randomBoolean() ? null : new CCSInfo("", randomBoolean()); - SearchShardTarget searchShardTarget = new SearchShardTarget("", new ShardId("", "", shardIndex), ccsInfo, OriginalIndices.NONE); + String clusterAlias = randomBoolean() ? null : "remote"; + SearchShardTarget searchShardTarget = new SearchShardTarget("", new ShardId("", "", shardIndex), + clusterAlias, OriginalIndices.NONE); QuerySearchResult querySearchResult = new QuerySearchResult(shardIndex, searchShardTarget); final TopDocs topDocs; float maxScore = 0; @@ -274,8 +274,8 @@ private static AtomicArray generateFetchResults(int nShards, AtomicArray fetchResults = new AtomicArray<>(nShards); for (int shardIndex = 0; shardIndex < nShards; shardIndex++) { float maxScore = -1F; - CCSInfo ccsInfo = randomBoolean() ? null : new CCSInfo("", randomBoolean()); - SearchShardTarget shardTarget = new SearchShardTarget("", new ShardId("", "", shardIndex), ccsInfo, OriginalIndices.NONE); + String clusterAlias = randomBoolean() ? null : "remote"; + SearchShardTarget shardTarget = new SearchShardTarget("", new ShardId("", "", shardIndex), clusterAlias, OriginalIndices.NONE); FetchSearchResult fetchSearchResult = new FetchSearchResult(shardIndex, shardTarget); List searchHits = new ArrayList<>(); for (ScoreDoc scoreDoc : mergedSearchDocs) { diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchShardIteratorTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchShardIteratorTests.java index a3ee739b81181..835a20dca9667 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchShardIteratorTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchShardIteratorTests.java @@ -22,7 +22,6 @@ import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.search.CCSInfo; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.test.ESTestCase; @@ -44,33 +43,22 @@ public void testGetOriginalIndices() { assertSame(originalIndices, searchShardIterator.getOriginalIndices()); } - public void testCCSInfo() { - CCSInfo ccsInfo = randomBoolean() ? null : new CCSInfo(randomAlphaOfLengthBetween(5, 10), randomBoolean()); + public void testGetClusterAlias() { + String clusterAlias = randomBoolean() ? null : randomAlphaOfLengthBetween(5, 10); ShardId shardId = new ShardId(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLength(10), randomInt()); - SearchShardIterator searchShardIterator = new SearchShardIterator(ccsInfo, shardId, Collections.emptyList(), OriginalIndices.NONE); - assertSame(ccsInfo, searchShardIterator.getCCSInfo()); - if (ccsInfo == null) { - assertNull(searchShardIterator.getCCSInfo()); - } else { - assertEquals(ccsInfo.getConnectionAlias(), searchShardIterator.getCCSInfo().getConnectionAlias()); - } + SearchShardIterator searchShardIterator = new SearchShardIterator(clusterAlias, shardId, Collections.emptyList(), OriginalIndices.NONE); + assertEquals(clusterAlias, searchShardIterator.getClusterAlias()); } public void testNewSearchShardTarget() { - CCSInfo ccsInfo = randomBoolean() ? null : new CCSInfo(randomAlphaOfLengthBetween(5, 10), randomBoolean()); + String clusterAlias = randomBoolean() ? null : randomAlphaOfLengthBetween(5, 10); ShardId shardId = new ShardId(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLength(10), randomInt()); OriginalIndices originalIndices = new OriginalIndices(new String[]{randomAlphaOfLengthBetween(3, 10)}, IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean())); - SearchShardIterator searchShardIterator = new SearchShardIterator(ccsInfo, shardId, Collections.emptyList(), originalIndices); + SearchShardIterator searchShardIterator = new SearchShardIterator(clusterAlias, shardId, Collections.emptyList(), originalIndices); String nodeId = randomAlphaOfLengthBetween(3, 10); SearchShardTarget searchShardTarget = searchShardIterator.newSearchShardTarget(nodeId); - if (ccsInfo == null) { - assertNull(searchShardTarget.getConnectionAlias()); - assertNull(searchShardTarget.getHitIndexPrefix()); - } else { - assertEquals(ccsInfo.getConnectionAlias(), searchShardTarget.getConnectionAlias()); - assertEquals(ccsInfo.getHitIndexPrefix(), searchShardTarget.getHitIndexPrefix()); - } + assertEquals(clusterAlias, searchShardTarget.getClusterAlias()); assertSame(shardId, searchShardTarget.getShardId()); assertEquals(nodeId, searchShardTarget.getNodeId()); assertSame(originalIndices, searchShardTarget.getOriginalIndices()); diff --git a/server/src/test/java/org/elasticsearch/action/search/ShardSearchFailureTests.java b/server/src/test/java/org/elasticsearch/action/search/ShardSearchFailureTests.java index c18ee4808672e..c4b27417ee2f7 100644 --- a/server/src/test/java/org/elasticsearch/action/search/ShardSearchFailureTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/ShardSearchFailureTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.action.search; -import org.elasticsearch.Version; import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.ParsingException; @@ -29,7 +28,6 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.search.CCSInfo; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; @@ -48,10 +46,9 @@ public static ShardSearchFailure createTestItem(String indexUuid, boolean transp if (randomBoolean()) { String nodeId = randomAlphaOfLengthBetween(5, 10); String indexName = randomAlphaOfLengthBetween(5, 10); - CCSInfo ccsInfo = randomBoolean() ? null : - new CCSInfo(randomAlphaOfLengthBetween(5, 10), transportSerialization && randomBoolean()); + String clusterAlias = randomBoolean() ? randomAlphaOfLengthBetween(5, 10) : null; searchShardTarget = new SearchShardTarget(nodeId, - new ShardId(new Index(indexName, indexUuid), randomInt()), ccsInfo, OriginalIndices.NONE); + new ShardId(new Index(indexName, indexUuid), randomInt()), clusterAlias, OriginalIndices.NONE); } return new ShardSearchFailure(ex, searchShardTarget); } @@ -123,8 +120,7 @@ public void testToXContent() throws IOException { public void testToXContentWithClusterAlias() throws IOException { ShardSearchFailure failure = new ShardSearchFailure(new ParsingException(0, 0, "some message", null), - new SearchShardTarget("nodeId", new ShardId(new Index("indexName", "indexUuid"), 123), - new CCSInfo("cluster1", randomBoolean()), OriginalIndices.NONE)); + new SearchShardTarget("nodeId", new ShardId(new Index("indexName", "indexUuid"), 123), "cluster1", OriginalIndices.NONE)); BytesReference xContent = toXContent(failure, XContentType.JSON, randomBoolean()); assertEquals( "{\"shard\":123," @@ -142,34 +138,12 @@ public void testToXContentWithClusterAlias() throws IOException { public void testSerialization() throws IOException { ShardSearchFailure testItem = createTestItem(randomAlphaOfLength(12), true); - Version version = VersionUtils.randomVersionBetween(random(), Version.V_7_0_0, Version.CURRENT); ShardSearchFailure deserializedInstance = copyStreamable(testItem, writableRegistry(), - ShardSearchFailure::new, version); + ShardSearchFailure::new, VersionUtils.randomVersion(random())); assertEquals(testItem.index(), deserializedInstance.index()); assertEquals(testItem.shard(), deserializedInstance.shard()); assertEquals(testItem.shardId(), deserializedInstance.shardId()); assertEquals(testItem.reason(), deserializedInstance.reason()); assertEquals(testItem.status(), deserializedInstance.status()); } - - //TODO rename and update version after backport - public void testSerializationPre7_0_0() throws IOException { - ShardSearchFailure testItem = createTestItem(randomAlphaOfLength(12), true); - Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, VersionUtils.getPreviousVersion(Version.V_7_0_0)); - ShardSearchFailure deserializedInstance = copyStreamable(testItem, writableRegistry(), ShardSearchFailure::new, version); - assertEquals(testItem.index(), deserializedInstance.index()); - if (testItem.shard() == null) { - assertNull(deserializedInstance.shard()); - } else { - //equality check on target doesn't work as CCSInfo is partially read (missing boolean localReduction flag set to false) - assertEquals(testItem.shard().getNodeId(), deserializedInstance.shard().getNodeId()); - assertEquals(testItem.shard().getShardId(), deserializedInstance.shard().getShardId()); - assertEquals(testItem.shard().getHitIndexPrefix(), deserializedInstance.shard().getHitIndexPrefix()); - assertEquals(testItem.shard().getHitIndexPrefix(), deserializedInstance.shard().getConnectionAlias()); - } - assertEquals(testItem.shardId(), deserializedInstance.shardId()); - assertEquals(testItem.reason(), deserializedInstance.reason()); - assertEquals(testItem.status(), deserializedInstance.status()); - } - } diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java index 079fc6c4da95c..085772368dc06 100644 --- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java @@ -36,7 +36,6 @@ import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.search.CCSInfo; import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.transport.MockTransportService; @@ -86,18 +85,17 @@ public void testMergeShardsIterators() { OriginalIndices remoteIndices = new OriginalIndices(new String[]{"remote_alias", "remote_index_2"}, IndicesOptions.strictExpandOpen()); List remoteShardIterators = new ArrayList<>(); - CCSInfo remoteCCSInfo = new CCSInfo("remote", false); { ShardId remoteShardId = new ShardId("remote_index", "remote_index_uuid", 2); ShardRouting remoteShardRouting = TestShardRouting.newShardRouting(remoteShardId, "remote_node", true, STARTED); - SearchShardIterator remoteShardIterator = new SearchShardIterator(remoteCCSInfo, remoteShardId, + SearchShardIterator remoteShardIterator = new SearchShardIterator("remote", remoteShardId, Collections.singletonList(remoteShardRouting), remoteIndices); remoteShardIterators.add(remoteShardIterator); } { ShardId remoteShardId2 = new ShardId("remote_index_2", "remote_index_2_uuid", 3); ShardRouting remoteShardRouting2 = TestShardRouting.newShardRouting(remoteShardId2, "remote_node", true, STARTED); - SearchShardIterator remoteShardIterator2 = new SearchShardIterator(remoteCCSInfo, remoteShardId2, + SearchShardIterator remoteShardIterator2 = new SearchShardIterator("remote", remoteShardId2, Collections.singletonList(remoteShardRouting2), remoteIndices); remoteShardIterators.add(remoteShardIterator2); } @@ -106,7 +104,7 @@ public void testMergeShardsIterators() { { ShardId remoteShardId3 = new ShardId("remote_index_3", "remote_index_3_uuid", 4); ShardRouting remoteShardRouting3 = TestShardRouting.newShardRouting(remoteShardId3, "remote_node", true, STARTED); - SearchShardIterator remoteShardIterator3 = new SearchShardIterator(remoteCCSInfo, remoteShardId3, + SearchShardIterator remoteShardIterator3 = new SearchShardIterator("remote", remoteShardId3, Collections.singletonList(remoteShardRouting3), remoteIndices2); remoteShardIterators.add(remoteShardIterator3); } @@ -123,54 +121,36 @@ public void testMergeShardsIterators() { assertEquals("local_index", searchShardIterator.shardId().getIndexName()); assertEquals(0, searchShardIterator.shardId().getId()); assertSame(localIndices, searchShardIterator.getOriginalIndices()); - assertLocalClusterAlias(searchShardIterator, localClusterAlias); + assertEquals(localClusterAlias, searchShardIterator.getClusterAlias()); break; case 1: assertEquals("local_index_2", searchShardIterator.shardId().getIndexName()); assertEquals(1, searchShardIterator.shardId().getId()); assertSame(localIndices, searchShardIterator.getOriginalIndices()); - assertLocalClusterAlias(searchShardIterator, localClusterAlias); + assertEquals(localClusterAlias, searchShardIterator.getClusterAlias()); break; case 2: assertEquals("remote_index", searchShardIterator.shardId().getIndexName()); assertEquals(2, searchShardIterator.shardId().getId()); assertSame(remoteIndices, searchShardIterator.getOriginalIndices()); - assertRemoteClusterAlias(searchShardIterator, remoteCCSInfo); + assertEquals("remote", searchShardIterator.getClusterAlias()); break; case 3: assertEquals("remote_index_2", searchShardIterator.shardId().getIndexName()); assertEquals(3, searchShardIterator.shardId().getId()); assertSame(remoteIndices, searchShardIterator.getOriginalIndices()); - assertRemoteClusterAlias(searchShardIterator, remoteCCSInfo); + assertEquals("remote", searchShardIterator.getClusterAlias()); break; case 4: assertEquals("remote_index_3", searchShardIterator.shardId().getIndexName()); assertEquals(4, searchShardIterator.shardId().getId()); assertSame(remoteIndices2, searchShardIterator.getOriginalIndices()); - assertRemoteClusterAlias(searchShardIterator, remoteCCSInfo); + assertEquals("remote", searchShardIterator.getClusterAlias()); break; } } } - private static void assertRemoteClusterAlias(SearchShardIterator searchShardIterator, CCSInfo ccsInfo) { - assertSame(ccsInfo, searchShardIterator.getCCSInfo()); - assertEquals("remote", searchShardIterator.getConnectionAlias()); - assertEquals("remote", searchShardIterator.getCCSInfo().getConnectionAlias()); - assertEquals("remote", searchShardIterator.getCCSInfo().getHitIndexPrefix()); - } - - private static void assertLocalClusterAlias(SearchShardIterator searchShardIterator, String localClusterAlias) { - if (localClusterAlias == null) { - assertNull(searchShardIterator.getCCSInfo()); - assertNull(searchShardIterator.getConnectionAlias()); - } else { - assertEquals(localClusterAlias, searchShardIterator.getCCSInfo().getHitIndexPrefix()); - assertNull(searchShardIterator.getCCSInfo().getConnectionAlias()); - assertNull(searchShardIterator.getConnectionAlias()); - } - } - public void testProcessRemoteShards() { try (TransportService transportService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { @@ -223,8 +203,7 @@ public void testProcessRemoteShards() { assertArrayEquals(new String[]{"some_alias_for_foo", "some_other_foo_alias"}, iterator.getOriginalIndices().indices()); assertTrue(iterator.shardId().getId() == 0 || iterator.shardId().getId() == 1); - assertEquals("test_cluster_1", iterator.getConnectionAlias()); - assertEquals(new CCSInfo("test_cluster_1", false), iterator.getCCSInfo()); + assertEquals("test_cluster_1", iterator.getClusterAlias()); assertEquals("foo", iterator.shardId().getIndexName()); ShardRouting shardRouting = iterator.nextOrNull(); assertNotNull(shardRouting); @@ -236,8 +215,7 @@ public void testProcessRemoteShards() { } else if (iterator.shardId().getIndexName().endsWith("bar")) { assertArrayEquals(new String[]{"bar"}, iterator.getOriginalIndices().indices()); assertEquals(0, iterator.shardId().getId()); - assertEquals("test_cluster_1", iterator.getConnectionAlias()); - assertEquals(new CCSInfo("test_cluster_1", false), iterator.getCCSInfo()); + assertEquals("test_cluster_1", iterator.getClusterAlias()); assertEquals("bar", iterator.shardId().getIndexName()); ShardRouting shardRouting = iterator.nextOrNull(); assertNotNull(shardRouting); @@ -250,8 +228,7 @@ public void testProcessRemoteShards() { assertArrayEquals(new String[]{"some_alias_for_xyz"}, iterator.getOriginalIndices().indices()); assertEquals(0, iterator.shardId().getId()); assertEquals("xyz", iterator.shardId().getIndexName()); - assertEquals("test_cluster_2", iterator.getConnectionAlias()); - assertEquals(new CCSInfo("test_cluster_2", false), iterator.getCCSInfo()); + assertEquals("test_cluster_2", iterator.getClusterAlias()); ShardRouting shardRouting = iterator.nextOrNull(); assertNotNull(shardRouting); assertEquals(shardRouting.getIndexName(), "xyz"); diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java index a84101da94cb9..163249c864524 100644 --- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java @@ -22,7 +22,6 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.search.CCSInfo; import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.test.ESTestCase; @@ -37,10 +36,10 @@ public void testParseScrollId() throws IOException { DiscoveryNode node2 = new DiscoveryNode("node_2", buildNewFakeTransportAddress(), Version.CURRENT); DiscoveryNode node3 = new DiscoveryNode("node_3", buildNewFakeTransportAddress(), Version.CURRENT); SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult1 = new SearchAsyncActionTests.TestSearchPhaseResult(1, node1); - SearchShardTarget target = new SearchShardTarget("node_1", new ShardId("idx", "uuid1", 2), new CCSInfo("cluster_x", false), null); + SearchShardTarget target = new SearchShardTarget("node_1", new ShardId("idx", "uuid1", 2), "cluster_x", null); testSearchPhaseResult1.setSearchShardTarget(target); SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult2 = new SearchAsyncActionTests.TestSearchPhaseResult(12, node2); - SearchShardTarget target2 = new SearchShardTarget("node_2", new ShardId("idy", "uuid2", 42), new CCSInfo("cluster_y", false), null); + SearchShardTarget target2 = new SearchShardTarget("node_2", new ShardId("idy", "uuid2", 42), "cluster_y", null); testSearchPhaseResult2.setSearchShardTarget(target2); SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult3 = new SearchAsyncActionTests.TestSearchPhaseResult(42, node3); testSearchPhaseResult3.setSearchShardTarget(new SearchShardTarget("node_3", new ShardId("idy", "uuid2", 43), null, null)); diff --git a/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java b/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java index d0f7ddcadc3c0..fc24bdf9691de 100644 --- a/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java +++ b/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java @@ -30,7 +30,6 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.Rewriteable; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.search.CCSInfo; import org.elasticsearch.search.Scroll; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.internal.AliasFilter; @@ -155,7 +154,7 @@ public Rewriteable getRewriteable() { } @Override - public CCSInfo getCCSInfo() { + public String getClusterAlias() { return null; } }; diff --git a/server/src/test/java/org/elasticsearch/rest/action/RestActionsTests.java b/server/src/test/java/org/elasticsearch/rest/action/RestActionsTests.java index 668962ccb68ec..d44b2b70a05ef 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/RestActionsTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/RestActionsTests.java @@ -36,7 +36,6 @@ import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.search.CCSInfo; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.test.ESTestCase; @@ -199,9 +198,8 @@ private static ShardSearchFailure createShardFailureParsingException(String node } private static SearchShardTarget createSearchShardTarget(String nodeId, int shardId, String index, String clusterAlias) { - CCSInfo ccsInfo = clusterAlias == null ? null : new CCSInfo(clusterAlias, randomBoolean()); return new SearchShardTarget(nodeId, - new ShardId(new Index(index, IndexMetaData.INDEX_UUID_NA_VALUE), shardId), ccsInfo, OriginalIndices.NONE); + new ShardId(new Index(index, IndexMetaData.INDEX_UUID_NA_VALUE), shardId), clusterAlias, OriginalIndices.NONE); } @Override diff --git a/server/src/test/java/org/elasticsearch/search/SearchHitTests.java b/server/src/test/java/org/elasticsearch/search/SearchHitTests.java index 50cb587df103d..fee55f1e22f23 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchHitTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchHitTests.java @@ -27,7 +27,6 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.document.DocumentField; -import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -46,7 +45,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Base64; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -128,9 +126,9 @@ public static SearchHit createTestItem(XContentType xContentType, boolean withOp } if (transportSerialization && randomBoolean()) { String index = randomAlphaOfLengthBetween(5, 10); - CCSInfo ccsInfo = randomBoolean() ? null : new CCSInfo(randomAlphaOfLengthBetween(5, 10), randomBoolean()); + String clusterAlias = randomBoolean() ? null : randomAlphaOfLengthBetween(5, 10); hit.shard(new SearchShardTarget(randomAlphaOfLengthBetween(5, 10), - new ShardId(new Index(index, randomAlphaOfLengthBetween(5, 10)), randomInt()), ccsInfo, OriginalIndices.NONE)); + new ShardId(new Index(index, randomAlphaOfLengthBetween(5, 10)), randomInt()), clusterAlias, OriginalIndices.NONE)); } return hit; } @@ -214,9 +212,9 @@ public void testToXContent() throws IOException { } public void testSerializeShardTarget() throws Exception { - CCSInfo ccsInfo = randomBoolean() ? null : new CCSInfo("cluster_alias", randomBoolean()); + String clusterAlias = randomBoolean() ? null : "cluster_alias"; SearchShardTarget target = new SearchShardTarget("_node_id", new ShardId(new Index("_index", "_na_"), 0), - ccsInfo, OriginalIndices.NONE); + clusterAlias, OriginalIndices.NONE); Map innerHits = new HashMap<>(); SearchHit innerHit1 = new SearchHit(0, "_id", new Text("_type"), null); @@ -245,23 +243,17 @@ public void testSerializeShardTarget() throws Exception { Version version = VersionUtils.randomVersion(random()); SearchHits results = copyStreamable(hits, getNamedWriteableRegistry(), SearchHits::new, version); SearchShardTarget deserializedTarget = results.getAt(0).getShard(); - //TODO update version after backport - if (version.before(Version.V_7_0_0) && ccsInfo != null) { - //localReduction flag is not serialized as part of CCSInfo - target = new SearchShardTarget(target.getNodeId(), target.getShardId(), - new CCSInfo(ccsInfo.getHitIndexPrefix(), false), target.getOriginalIndices()); - } assertThat(deserializedTarget, equalTo(target)); assertThat(results.getAt(0).getInnerHits().get("1").getAt(0).getShard(), notNullValue()); assertThat(results.getAt(0).getInnerHits().get("1").getAt(0).getInnerHits().get("1").getAt(0).getShard(), notNullValue()); assertThat(results.getAt(0).getInnerHits().get("1").getAt(1).getShard(), notNullValue()); assertThat(results.getAt(0).getInnerHits().get("2").getAt(0).getShard(), notNullValue()); for (SearchHit hit : results) { - assertEquals(ccsInfo == null ? null : ccsInfo.getHitIndexPrefix(), hit.getClusterAlias()); + assertEquals(clusterAlias, hit.getClusterAlias()); if (hit.getInnerHits() != null) { for (SearchHits innerhits : hit.getInnerHits().values()) { for (SearchHit innerHit : innerhits) { - assertEquals(ccsInfo == null ? null : ccsInfo.getHitIndexPrefix(), innerHit.getClusterAlias()); + assertEquals(clusterAlias, innerHit.getClusterAlias()); } } } @@ -269,30 +261,6 @@ public void testSerializeShardTarget() throws Exception { assertThat(results.getAt(1).getShard(), equalTo(target)); } - //TODO rename and update version after backport - public void testReadFromPre7_0_0() throws IOException { - String msg = "/4AAAAAAAAJpZP////8A//////////8AAAAAAAAAAQEAAAAEbm9kZQVpbmRleAR1dWlkAAEOcmVtb3RlX2NsdXN0ZXIBA2tl" + - "eQEAf8AAAAAAAAAAAAAAAAAAAAAAAAAA"; - SearchHit hit = new SearchHit(); - try (StreamInput in = StreamInput.wrap(Base64.getDecoder().decode(msg))) { - //TODO randomize the version after backport, conflicts with SearchSortValues wire changes otherwise - in.setVersion(Version.V_6_6_0); - hit.readFrom(in); - } - assertEquals("id", hit.getId()); - assertEquals(0, hit.docId()); - SearchShardTarget shardTarget = hit.getShard(); - assertEquals("node", shardTarget.getNodeId()); - assertEquals(new ShardId(new Index("index", "uuid"), 0), shardTarget.getShardId()); - assertEquals("remote_cluster", shardTarget.getHitIndexPrefix()); - assertEquals("remote_cluster", shardTarget.getConnectionAlias()); - Map innerHits = hit.getInnerHits(); - assertEquals(1, innerHits.size()); - SearchHits searchHits = innerHits.get("key"); - assertEquals(0, searchHits.getHits().length); - assertEquals(0, searchHits.getTotalHits().value); - } - public void testNullSource() { SearchHit searchHit = new SearchHit(0, "_id", new Text("_type"), null); diff --git a/server/src/test/java/org/elasticsearch/search/SearchHitsTests.java b/server/src/test/java/org/elasticsearch/search/SearchHitsTests.java index 2239d659f45e4..8623c2021bffd 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchHitsTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchHitsTests.java @@ -232,9 +232,9 @@ public void testFromXContentWithShards() throws IOException { for (SearchHit hit : hits) { String index = randomAlphaOfLengthBetween(5, 10); - CCSInfo ccsInfo = randomBoolean() ? null : new CCSInfo(randomAlphaOfLengthBetween(5, 10), randomBoolean()); + String clusterAlias = randomBoolean() ? null : randomAlphaOfLengthBetween(5, 10); final SearchShardTarget shardTarget = new SearchShardTarget(randomAlphaOfLengthBetween(5, 10), - new ShardId(new Index(index, randomAlphaOfLengthBetween(5, 10)), randomInt()), ccsInfo, OriginalIndices.NONE); + new ShardId(new Index(index, randomAlphaOfLengthBetween(5, 10)), randomInt()), clusterAlias, OriginalIndices.NONE); if (withExplanation) { hit.explanation(SearchHitTests.createExplanation(randomIntBetween(0, 5))); } @@ -291,15 +291,6 @@ public void testSerializationPre6_6_0() throws IOException { Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, VersionUtils.getPreviousVersion(Version.V_6_6_0)); SearchHits original = createTestItem(randomFrom(XContentType.values()), false, true, TotalHits.Relation.EQUAL_TO); SearchHits deserialized = copyInstance(original, version); - for (SearchHit originalHit : original.getHits()) { - SearchShardTarget shard = originalHit.getShard(); - if (shard != null && shard.getHitIndexPrefix() != null) { - //CCSInfo is not fully read, localReduction is always set to false when reading from previous versions - SearchShardTarget newTarget = new SearchShardTarget(shard.getNodeId(), shard.getShardId(), - new CCSInfo(shard.getHitIndexPrefix(), false), shard.getOriginalIndices()); - originalHit.shard(newTarget); - } - } assertArrayEquals(original.getHits(), deserialized.getHits()); assertEquals(original.getMaxScore(), deserialized.getMaxScore(), 0F); if (original.getTotalHits() == null) { diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java index 4116c05abc940..894a4fa9d4ae3 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java @@ -646,24 +646,18 @@ public void testCreateSearchContext() throws IOException { final SearchService service = getInstanceFromNode(SearchService.class); ShardId shardId = new ShardId(indexService.index(), 0); long nowInMillis = System.currentTimeMillis(); - CCSInfo ccsInfo = randomBoolean() ? null : new CCSInfo(randomAlphaOfLengthBetween(3, 10), randomBoolean()); + String clusterAlias = randomBoolean() ? null : randomAlphaOfLengthBetween(3, 10); SearchRequest searchRequest = new SearchRequest(); searchRequest.allowPartialSearchResults(randomBoolean()); ShardSearchTransportRequest request = new ShardSearchTransportRequest(OriginalIndices.NONE, searchRequest, shardId, - indexService.numberOfShards(), AliasFilter.EMPTY, 1f, nowInMillis, ccsInfo, Strings.EMPTY_ARRAY); + indexService.numberOfShards(), AliasFilter.EMPTY, 1f, nowInMillis, clusterAlias, Strings.EMPTY_ARRAY); DefaultSearchContext searchContext = service.createSearchContext(request, new TimeValue(System.currentTimeMillis())); SearchShardTarget searchShardTarget = searchContext.shardTarget(); QueryShardContext queryShardContext = searchContext.getQueryShardContext(); - String expectedIndexName = ccsInfo == null ? index : ccsInfo.getHitIndexPrefix() + ":" + index; + String expectedIndexName = clusterAlias == null ? index : clusterAlias + ":" + index; assertEquals(expectedIndexName, queryShardContext.getFullyQualifiedIndex().getName()); assertEquals(expectedIndexName, searchShardTarget.getFullyQualifiedIndexName()); - if (ccsInfo == null) { - assertNull(searchShardTarget.getHitIndexPrefix()); - assertNull(searchShardTarget.getConnectionAlias()); - } else { - assertEquals(ccsInfo.getHitIndexPrefix(), searchShardTarget.getHitIndexPrefix()); - assertEquals(ccsInfo.getConnectionAlias(), searchShardTarget.getConnectionAlias()); - } + assertEquals(clusterAlias, searchShardTarget.getClusterAlias()); assertEquals(shardId, searchShardTarget.getShardId()); assertSame(searchShardTarget, searchContext.dfsResult().getSearchShardTarget()); assertSame(searchShardTarget, searchContext.queryResult().getSearchShardTarget()); diff --git a/server/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java b/server/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java index f9ee772fccf3d..da987a657260a 100644 --- a/server/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java +++ b/server/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java @@ -28,7 +28,6 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; -import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.ToXContent; @@ -41,11 +40,8 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.InvalidAliasNameException; import org.elasticsearch.search.AbstractSearchTestCase; -import org.elasticsearch.search.CCSInfo; -import org.elasticsearch.test.VersionUtils; import java.io.IOException; -import java.util.Base64; import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; import static org.elasticsearch.index.query.QueryBuilders.termQuery; @@ -79,31 +75,7 @@ public void testSerialization() throws Exception { assertNotSame(deserializedRequest, shardSearchTransportRequest); assertEquals(deserializedRequest.getAliasFilter(), shardSearchTransportRequest.getAliasFilter()); assertEquals(deserializedRequest.indexBoost(), shardSearchTransportRequest.indexBoost(), 0.0f); - assertEquals(deserializedRequest.getCCSInfo(), shardSearchTransportRequest.getCCSInfo()); - } - - //TODO rename and update version after backport - public void testSerializationPre7_0_0() throws IOException { - Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, VersionUtils.getPreviousVersion(Version.V_7_0_0)); - ShardSearchTransportRequest shardSearchTransportRequest = createShardSearchTransportRequest(); - ShardSearchTransportRequest deserializedRequest = - copyWriteable(shardSearchTransportRequest, namedWriteableRegistry, ShardSearchTransportRequest::new, version); - assertEquals(new CCSInfo(shardSearchTransportRequest.getCCSInfo().getHitIndexPrefix(), false), deserializedRequest.getCCSInfo()); - } - - //TODO rename and update version after backport - public void testReadFromPre7_0_0() throws IOException { - String msg = "AAVpbmRleAR1dWlkAAEBAAEAAgAAAP////8PAAAAAAAAAP////8PAAAAAAAAAgAAAAAAAQAAAD+" + - "AAAAAAgEGcmVtb3RlAAAAAQVpbmRleAMCBAUBAAAAAAAAAAAAAAAA"; - try (StreamInput in = StreamInput.wrap(Base64.getDecoder().decode(msg))) { - in.setVersion(VersionUtils.randomVersionBetween(random(), Version.V_6_4_0, VersionUtils.getPreviousVersion(Version.V_7_0_0))); - ShardSearchTransportRequest request = new ShardSearchTransportRequest(in); - assertArrayEquals(new String[]{"index"}, request.indices()); - assertEquals(new ShardId("index", "uuid", 0), request.shardId()); - assertEquals(1, request.numberOfShards()); - assertEquals("remote", request.getCCSInfo().getHitIndexPrefix()); - assertEquals("remote", request.getCCSInfo().getConnectionAlias()); - } + assertEquals(deserializedRequest.getClusterAlias(), shardSearchTransportRequest.getClusterAlias()); } private ShardSearchTransportRequest createShardSearchTransportRequest() throws IOException { @@ -119,7 +91,7 @@ private ShardSearchTransportRequest createShardSearchTransportRequest() throws I final String[] routings = generateRandomStringArray(5, 10, false, true); return new ShardSearchTransportRequest(new OriginalIndices(searchRequest), searchRequest, shardId, randomIntBetween(1, 100), filteringAliases, randomBoolean() ? 1.0f : randomFloat(), - Math.abs(randomLong()), new CCSInfo(randomAlphaOfLengthBetween(3, 10), randomBoolean()), routings); + Math.abs(randomLong()), randomAlphaOfLengthBetween(3, 10), routings); } public void testFilteringAliases() throws Exception { @@ -176,8 +148,7 @@ public static CompressedXContent filter(QueryBuilder filterBuilder) throws IOExc } private IndexMetaData remove(IndexMetaData indexMetaData, String alias) { - IndexMetaData build = IndexMetaData.builder(indexMetaData).removeAlias(alias).build(); - return build; + return IndexMetaData.builder(indexMetaData).removeAlias(alias).build(); } private IndexMetaData add(IndexMetaData indexMetaData, String alias, @Nullable CompressedXContent filter) { diff --git a/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java b/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java index 8d996a7a1b87c..30ed0cb5ab5b5 100644 --- a/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java @@ -58,7 +58,6 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.Rewriteable; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.search.CCSInfo; import org.elasticsearch.search.Scroll; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.internal.AliasFilter; @@ -199,7 +198,7 @@ public BytesReference cacheKey() throws IOException { } @Override - public CCSInfo getCCSInfo() { + public String getClusterAlias() { return null; } From be9d8fa8d78cb39c87c36103c717089c96361440 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Thu, 27 Dec 2018 16:00:45 +0100 Subject: [PATCH 3/9] add package private constructor --- .../action/search/SearchRequest.java | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java index f5e0eac8e2681..2c84f9d4cb613 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -62,6 +62,8 @@ public final class SearchRequest extends ActionRequest implements IndicesRequest public static final int DEFAULT_PRE_FILTER_SHARD_SIZE = 128; public static final int DEFAULT_BATCHED_REDUCE_SIZE = 512; + private final String clusterAlias; + private SearchType searchType = SearchType.DEFAULT; private String[] indices = Strings.EMPTY_ARRAY; @@ -91,9 +93,8 @@ public final class SearchRequest extends ActionRequest implements IndicesRequest private IndicesOptions indicesOptions = DEFAULT_INDICES_OPTIONS; - private String clusterAlias; - public SearchRequest() { + this.clusterAlias = null; } /** @@ -128,6 +129,7 @@ public SearchRequest(String... indices) { * Constructs a new search request against the provided indices with the given search source. */ public SearchRequest(String[] indices, SearchSourceBuilder source) { + this(); if (source == null) { throw new IllegalArgumentException("source must not be null"); } @@ -135,6 +137,15 @@ public SearchRequest(String[] indices, SearchSourceBuilder source) { this.source = source; } + /** + * Creates a new search request by providing the alias of the cluster where it will be executed. Used when a {@link SearchRequest} + * is created and executed as part of a cross-cluster search request performing local reduction on each cluster. + * The coordinating CCS node provides the alias to prefix index names with in the returned search results. + */ + SearchRequest(String clusterAlias) { + this.clusterAlias = Objects.requireNonNull(clusterAlias, "cluster alias must not be null"); + } + /** * Constructs a new search request from reading the specified stream. * @@ -164,6 +175,8 @@ public SearchRequest(StreamInput in) throws IOException { //TODO update version after backport if (in.getVersion().onOrAfter(Version.V_7_0_0)) { clusterAlias = in.readOptionalString(); + } else { + clusterAlias = null; } } @@ -230,14 +243,6 @@ String getClusterAlias() { return clusterAlias; } - /** - * Sets the cluster alias for this search request. Used when a {@link SearchRequest} is executed as part of a cross-cluster search - * request performing local reduction on each cluster. This way the coordinating CCS node provides the alias to prefix index names with. - */ - void setClusterAlias(String clusterAlias) { - this.clusterAlias = clusterAlias; - } - /** * Sets the indices the search will be executed on. */ From 3d5d686d17b8d36ec3fb80734bccef54b9c394d0 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Thu, 27 Dec 2018 16:12:26 +0100 Subject: [PATCH 4/9] remove needless changes --- .../org/elasticsearch/action/search/FetchSearchPhase.java | 3 +-- .../org/elasticsearch/action/search/ShardSearchFailure.java | 4 ++-- .../elasticsearch/action/search/TransportSearchHelper.java | 5 ++--- .../elasticsearch/search/internal/ShardSearchRequest.java | 3 +++ .../action/search/ShardSearchFailureTests.java | 6 +++--- .../action/search/TransportSearchHelperTests.java | 6 ++---- 6 files changed, 13 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java index a86d638abe342..1bbca35cb9a54 100644 --- a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java @@ -190,8 +190,7 @@ private void releaseIrrelevantSearchContext(QuerySearchResult queryResult) { if (context.getRequest().scroll() == null && queryResult.hasSearchContext()) { try { SearchShardTarget searchShardTarget = queryResult.getSearchShardTarget(); - Transport.Connection connection = context.getConnection(searchShardTarget.getClusterAlias(), - searchShardTarget.getNodeId()); + Transport.Connection connection = context.getConnection(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId()); context.sendReleaseSearchContext(queryResult.getRequestId(), connection, searchShardTarget.getOriginalIndices()); } catch (Exception e) { context.getLogger().trace("failed to release context", e); diff --git a/server/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java b/server/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java index 7927d4b0f6278..451ceda70fdc4 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java +++ b/server/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java @@ -169,8 +169,8 @@ public static ShardSearchFailure fromXContent(XContentParser parser) throws IOEx } SearchShardTarget searchShardTarget = null; if (nodeId != null) { - searchShardTarget = new SearchShardTarget(nodeId, new ShardId(new Index(indexName, IndexMetaData.INDEX_UUID_NA_VALUE), shardId), - clusterAlias, OriginalIndices.NONE); + searchShardTarget = new SearchShardTarget(nodeId, + new ShardId(new Index(indexName, IndexMetaData.INDEX_UUID_NA_VALUE), shardId), clusterAlias, OriginalIndices.NONE); } return new ShardSearchFailure(exception, searchShardTarget); } diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java index 3a4e01c09c9af..7044ffdb27d5d 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java @@ -43,9 +43,8 @@ static String buildScrollId(AtomicArray searchPhase for (SearchPhaseResult searchPhaseResult : searchPhaseResults.asList()) { out.writeLong(searchPhaseResult.getRequestId()); SearchShardTarget searchShardTarget = searchPhaseResult.getSearchShardTarget(); - String hitIndexPrefix = searchShardTarget.getClusterAlias(); - if (hitIndexPrefix != null) { - out.writeString(RemoteClusterAware.buildRemoteIndexName(hitIndexPrefix, searchShardTarget.getNodeId())); + if (searchShardTarget.getClusterAlias() != null) { + out.writeString(RemoteClusterAware.buildRemoteIndexName(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId())); } else { out.writeString(searchShardTarget.getNodeId()); } diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java index 2de73af1a1b5c..3fc16584eb0bf 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java @@ -151,6 +151,9 @@ static QueryBuilder parseAliasFilter(CheckedFunctionnull otherwise. + */ @Nullable String getClusterAlias(); diff --git a/server/src/test/java/org/elasticsearch/action/search/ShardSearchFailureTests.java b/server/src/test/java/org/elasticsearch/action/search/ShardSearchFailureTests.java index c4b27417ee2f7..f62f874c9e2c9 100644 --- a/server/src/test/java/org/elasticsearch/action/search/ShardSearchFailureTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/ShardSearchFailureTests.java @@ -39,7 +39,7 @@ public class ShardSearchFailureTests extends ESTestCase { - public static ShardSearchFailure createTestItem(String indexUuid, boolean transportSerialization) { + public static ShardSearchFailure createTestItem(String indexUuid) { String randomMessage = randomAlphaOfLengthBetween(3, 20); Exception ex = new ParsingException(0, 0, randomMessage , new IllegalArgumentException("some bad argument")); SearchShardTarget searchShardTarget = null; @@ -67,7 +67,7 @@ public void testFromXContentWithRandomFields() throws IOException { } private void doFromXContentTestWithRandomFields(boolean addRandomFields) throws IOException { - ShardSearchFailure response = createTestItem(IndexMetaData.INDEX_UUID_NA_VALUE, false); + ShardSearchFailure response = createTestItem(IndexMetaData.INDEX_UUID_NA_VALUE); XContentType xContentType = randomFrom(XContentType.values()); boolean humanReadable = randomBoolean(); BytesReference originalBytes = toShuffledXContent(response, xContentType, ToXContent.EMPTY_PARAMS, humanReadable); @@ -137,7 +137,7 @@ public void testToXContentWithClusterAlias() throws IOException { } public void testSerialization() throws IOException { - ShardSearchFailure testItem = createTestItem(randomAlphaOfLength(12), true); + ShardSearchFailure testItem = createTestItem(randomAlphaOfLength(12)); ShardSearchFailure deserializedInstance = copyStreamable(testItem, writableRegistry(), ShardSearchFailure::new, VersionUtils.randomVersion(random())); assertEquals(testItem.index(), deserializedInstance.index()); diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java index 163249c864524..a666f45c34311 100644 --- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchHelperTests.java @@ -36,11 +36,9 @@ public void testParseScrollId() throws IOException { DiscoveryNode node2 = new DiscoveryNode("node_2", buildNewFakeTransportAddress(), Version.CURRENT); DiscoveryNode node3 = new DiscoveryNode("node_3", buildNewFakeTransportAddress(), Version.CURRENT); SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult1 = new SearchAsyncActionTests.TestSearchPhaseResult(1, node1); - SearchShardTarget target = new SearchShardTarget("node_1", new ShardId("idx", "uuid1", 2), "cluster_x", null); - testSearchPhaseResult1.setSearchShardTarget(target); + testSearchPhaseResult1.setSearchShardTarget(new SearchShardTarget("node_1", new ShardId("idx", "uuid1", 2), "cluster_x", null)); SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult2 = new SearchAsyncActionTests.TestSearchPhaseResult(12, node2); - SearchShardTarget target2 = new SearchShardTarget("node_2", new ShardId("idy", "uuid2", 42), "cluster_y", null); - testSearchPhaseResult2.setSearchShardTarget(target2); + testSearchPhaseResult2.setSearchShardTarget(new SearchShardTarget("node_2", new ShardId("idy", "uuid2", 42), "cluster_y", null)); SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult3 = new SearchAsyncActionTests.TestSearchPhaseResult(42, node3); testSearchPhaseResult3.setSearchShardTarget(new SearchShardTarget("node_3", new ShardId("idy", "uuid2", 43), null, null)); array.setOnce(0, testSearchPhaseResult1); From 1384f4f18c6a0e3386de06cbfeaf9b737dfc9b9b Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Thu, 27 Dec 2018 16:23:55 +0100 Subject: [PATCH 5/9] fix test --- .../action/search/SearchShardIterator.java | 2 +- .../action/search/SearchRequestTests.java | 10 ++++++---- .../action/search/SearchResponseTests.java | 2 +- .../search/RandomSearchRequestGenerator.java | 12 +++++++++++- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java b/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java index 3e1d2ddbbdd97..be3b5d7a9c2b3 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java @@ -63,7 +63,7 @@ public OriginalIndices getOriginalIndices() { } /** - * Returns the alias of the cluster where the shard is located + * Returns the alias of the cluster where the shard is located. */ @Nullable public String getClusterAlias() { diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java index ce130f388015d..c640390708b6a 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.util.ArrayUtils; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.AbstractSearchTestCase; +import org.elasticsearch.search.RandomSearchRequestGenerator; import org.elasticsearch.search.Scroll; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.rescore.QueryRescorerBuilder; @@ -44,9 +45,12 @@ public class SearchRequestTests extends AbstractSearchTestCase { @Override protected SearchRequest createSearchRequest() throws IOException { - SearchRequest searchRequest = super.createSearchRequest(); + if (randomBoolean()) { + return super.createSearchRequest(); + } //clusterAlias does not have public getter/setter hence we randomize it only in this test specifically. - searchRequest.setClusterAlias(randomAlphaOfLengthBetween(5, 10)); + SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(5, 10)); + RandomSearchRequestGenerator.randomSearchRequest(searchRequest, this::createSearchSourceBuilder); return searchRequest; } @@ -192,8 +196,6 @@ private SearchRequest mutate(SearchRequest searchRequest) { mutators.add(() -> mutation.searchType(randomValueOtherThan(searchRequest.searchType(), () -> randomFrom(SearchType.DFS_QUERY_THEN_FETCH, SearchType.QUERY_THEN_FETCH)))); mutators.add(() -> mutation.source(randomValueOtherThan(searchRequest.source(), this::createSearchSourceBuilder))); - mutators.add(() -> mutation.setClusterAlias(randomValueOtherThan(searchRequest.getClusterAlias(), - () -> randomAlphaOfLengthBetween(3, 10)))); randomFrom(mutators).run(); return mutation; } diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java index 65a37d0c8686c..ecd2dc44a702e 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java @@ -182,7 +182,7 @@ public void testFromXContentWithFailures() throws IOException { int numFailures = randomIntBetween(1, 5); ShardSearchFailure[] failures = new ShardSearchFailure[numFailures]; for (int i = 0; i < failures.length; i++) { - failures[i] = ShardSearchFailureTests.createTestItem(IndexMetaData.INDEX_UUID_NA_VALUE, false); + failures[i] = ShardSearchFailureTests.createTestItem(IndexMetaData.INDEX_UUID_NA_VALUE); } SearchResponse response = createTestItem(failures); XContentType xcontentType = randomFrom(XContentType.values()); diff --git a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java index d534af5789448..279bddebc4a28 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java @@ -83,7 +83,17 @@ private RandomSearchRequestGenerator() {} * {@link #randomSearchSourceBuilder(Supplier, Supplier, Supplier, Supplier, Supplier)}. */ public static SearchRequest randomSearchRequest(Supplier randomSearchSourceBuilder) { - SearchRequest searchRequest = new SearchRequest(); + return randomSearchRequest(new SearchRequest(), randomSearchSourceBuilder); + } + + /** + * Set random fields to the provided search request. + * + * @param searchRequest the search request + * @param randomSearchSourceBuilder builds a random {@link SearchSourceBuilder}. You can use + * {@link #randomSearchSourceBuilder(Supplier, Supplier, Supplier, Supplier, Supplier)}. + */ + public static SearchRequest randomSearchRequest(SearchRequest searchRequest, Supplier randomSearchSourceBuilder) { searchRequest.allowPartialSearchResults(true); if (randomBoolean()) { searchRequest.indices(generateRandomStringArray(10, 10, false, false)); From 865409d029febbca12ca682420e0a54ca381b146 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Thu, 27 Dec 2018 16:57:11 +0100 Subject: [PATCH 6/9] checkstyle --- .../org/elasticsearch/action/search/TransportSearchHelper.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java index 7044ffdb27d5d..c848e227af4c0 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java @@ -44,7 +44,8 @@ static String buildScrollId(AtomicArray searchPhase out.writeLong(searchPhaseResult.getRequestId()); SearchShardTarget searchShardTarget = searchPhaseResult.getSearchShardTarget(); if (searchShardTarget.getClusterAlias() != null) { - out.writeString(RemoteClusterAware.buildRemoteIndexName(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId())); + out.writeString( + RemoteClusterAware.buildRemoteIndexName(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId())); } else { out.writeString(searchShardTarget.getNodeId()); } From 5aa6b1396d0770258d42aad1033bd474aa58f9e5 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Thu, 27 Dec 2018 20:18:58 +0100 Subject: [PATCH 7/9] checkstyle --- .../elasticsearch/action/search/SearchShardIteratorTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchShardIteratorTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchShardIteratorTests.java index 835a20dca9667..09595650932c5 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchShardIteratorTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchShardIteratorTests.java @@ -46,7 +46,8 @@ public void testGetOriginalIndices() { public void testGetClusterAlias() { String clusterAlias = randomBoolean() ? null : randomAlphaOfLengthBetween(5, 10); ShardId shardId = new ShardId(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLength(10), randomInt()); - SearchShardIterator searchShardIterator = new SearchShardIterator(clusterAlias, shardId, Collections.emptyList(), OriginalIndices.NONE); + SearchShardIterator searchShardIterator = new SearchShardIterator(clusterAlias, shardId, Collections.emptyList(), + OriginalIndices.NONE); assertEquals(clusterAlias, searchShardIterator.getClusterAlias()); } From ebec06abab5fecca6a24cd027fc8e4641423b8bd Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Thu, 27 Dec 2018 21:38:52 +0100 Subject: [PATCH 8/9] add test for connection lookup --- .../action/search/TransportSearchAction.java | 30 ++++++---- .../search/TransportSearchActionTests.java | 59 +++++++++++++++++++ 2 files changed, 79 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index a74a33473cd1e..b866dc6764aed 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -61,6 +61,7 @@ import java.util.Set; import java.util.concurrent.Executor; import java.util.function.BiFunction; +import java.util.function.Function; import java.util.function.LongSupplier; import static org.elasticsearch.action.search.SearchType.QUERY_THEN_FETCH; @@ -339,25 +340,34 @@ private void executeSearch(SearchTask task, SearchTimeProvider timeProvider, Sea } final DiscoveryNodes nodes = clusterState.nodes(); - BiFunction connectionLookup = (clusterName, nodeId) -> { + BiFunction connectionLookup = buildConnectionLookup(searchRequest.getClusterAlias(), + nodes::get, remoteConnections, searchTransportService::getConnection); + boolean preFilterSearchShards = shouldPreFilterSearchShards(searchRequest, shardIterators); + searchAsyncAction(task, searchRequest, shardIterators, timeProvider, connectionLookup, clusterState.version(), + Collections.unmodifiableMap(aliasFilter), concreteIndexBoosts, routingMap, listener, preFilterSearchShards, clusters).start(); + } + + static BiFunction buildConnectionLookup(String requestClusterAlias, + Function localNodes, + BiFunction remoteNodes, + BiFunction nodeToConnection) { + return (clusterAlias, nodeId) -> { final DiscoveryNode discoveryNode; - if (clusterName == null || searchRequest.getClusterAlias() != null) { - assert searchRequest.getClusterAlias() == null || searchRequest.getClusterAlias().equals(clusterName); - discoveryNode = nodes.get(nodeId); + if (clusterAlias == null || requestClusterAlias != null) { + assert requestClusterAlias == null || requestClusterAlias.equals(clusterAlias); + discoveryNode = localNodes.apply(nodeId); } else { - discoveryNode = remoteConnections.apply(clusterName, nodeId); + discoveryNode = remoteNodes.apply(clusterAlias, nodeId); } if (discoveryNode == null) { throw new IllegalStateException("no node found for id: " + nodeId); } - return searchTransportService.getConnection(clusterName, discoveryNode); + return nodeToConnection.apply(clusterAlias, discoveryNode); }; - boolean preFilterSearchShards = shouldPreFilterSearchShards(searchRequest, shardIterators); - searchAsyncAction(task, searchRequest, shardIterators, timeProvider, connectionLookup, clusterState.version(), - Collections.unmodifiableMap(aliasFilter), concreteIndexBoosts, routingMap, listener, preFilterSearchShards, clusters).start(); } - private boolean shouldPreFilterSearchShards(SearchRequest searchRequest, GroupShardsIterator shardIterators) { + private static boolean shouldPreFilterSearchShards(SearchRequest searchRequest, + GroupShardsIterator shardIterators) { SearchSourceBuilder source = searchRequest.source(); return searchRequest.searchType() == QUERY_THEN_FETCH && // we can't do this for DFS it needs to fan out to all shards all the time SearchService.canRewriteToMatchNone(source) && diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java index 085772368dc06..16ff4389d7c4a 100644 --- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.action.search; import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsGroup; import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse; @@ -33,6 +34,7 @@ import org.elasticsearch.cluster.routing.TestShardRouting; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.index.shard.ShardId; @@ -42,6 +44,10 @@ import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.RemoteClusterService; +import org.elasticsearch.transport.Transport; +import org.elasticsearch.transport.TransportException; +import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportService; import java.util.ArrayList; @@ -50,8 +56,11 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.function.BiFunction; +import java.util.function.Function; import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; +import static org.hamcrest.CoreMatchers.startsWith; public class TransportSearchActionTests extends ESTestCase { @@ -245,6 +254,56 @@ public void testProcessRemoteShards() { } } + public void testBuildConnectionLookup() { + Function localNodes = (nodeId) -> new DiscoveryNode("local-" + nodeId, + new TransportAddress(TransportAddress.META_ADDRESS, 1024), Version.CURRENT); + BiFunction remoteNodes = (clusterAlias, nodeId) -> new DiscoveryNode("remote-" + nodeId, + new TransportAddress(TransportAddress.META_ADDRESS, 2048), Version.CURRENT); + BiFunction nodeToConnection = (clusterAlias, node) -> new Transport.Connection() { + @Override + public DiscoveryNode getNode() { + return node; + } + + @Override + public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) + throws TransportException { + } + + @Override + public void addCloseListener(ActionListener listener) { + } + + @Override + public boolean isClosed() { + return false; + } + + @Override + public void close() { + } + }; + + { + BiFunction connectionLookup = TransportSearchAction.buildConnectionLookup( + null, localNodes, remoteNodes, nodeToConnection); + + Transport.Connection localConnection = connectionLookup.apply(null, randomAlphaOfLengthBetween(5, 10)); + assertThat(localConnection.getNode().getId(), startsWith("local-")); + Transport.Connection remoteConnection = connectionLookup.apply(randomAlphaOfLengthBetween(5, 10), + randomAlphaOfLengthBetween(5, 10)); + assertThat(remoteConnection.getNode().getId(), startsWith("remote-")); + } + { + String requestClusterAlias = randomAlphaOfLengthBetween(5, 10); + BiFunction connectionLookup = TransportSearchAction.buildConnectionLookup( + requestClusterAlias, localNodes, remoteNodes, nodeToConnection); + + Transport.Connection localConnection = connectionLookup.apply(requestClusterAlias, randomAlphaOfLengthBetween(5, 10)); + assertThat(localConnection.getNode().getId(), startsWith("local-")); + } + } + public void testBuildClusters() { OriginalIndices localIndices = randomBoolean() ? null : randomOriginalIndices(); Map remoteIndices = new HashMap<>(); From 58fe6a8a3fb4733195d850ee21f3237a1ce98dd0 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Fri, 28 Dec 2018 09:14:30 +0100 Subject: [PATCH 9/9] rename clusterAlias to localClusterAlias --- .../action/search/SearchRequest.java | 30 +++++++++---------- .../action/search/TransportSearchAction.java | 4 +-- .../action/search/SearchRequestTests.java | 6 ++-- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java index 2c84f9d4cb613..9789e03c83641 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -62,7 +62,7 @@ public final class SearchRequest extends ActionRequest implements IndicesRequest public static final int DEFAULT_PRE_FILTER_SHARD_SIZE = 128; public static final int DEFAULT_BATCHED_REDUCE_SIZE = 512; - private final String clusterAlias; + private final String localClusterAlias; private SearchType searchType = SearchType.DEFAULT; @@ -94,7 +94,7 @@ public final class SearchRequest extends ActionRequest implements IndicesRequest private IndicesOptions indicesOptions = DEFAULT_INDICES_OPTIONS; public SearchRequest() { - this.clusterAlias = null; + this.localClusterAlias = null; } /** @@ -114,7 +114,7 @@ public SearchRequest(SearchRequest searchRequest) { this.searchType = searchRequest.searchType; this.source = searchRequest.source; this.types = searchRequest.types; - this.clusterAlias = searchRequest.clusterAlias; + this.localClusterAlias = searchRequest.localClusterAlias; } /** @@ -142,8 +142,8 @@ public SearchRequest(String[] indices, SearchSourceBuilder source) { * is created and executed as part of a cross-cluster search request performing local reduction on each cluster. * The coordinating CCS node provides the alias to prefix index names with in the returned search results. */ - SearchRequest(String clusterAlias) { - this.clusterAlias = Objects.requireNonNull(clusterAlias, "cluster alias must not be null"); + SearchRequest(String localClusterAlias) { + this.localClusterAlias = Objects.requireNonNull(localClusterAlias, "cluster alias must not be null"); } /** @@ -174,9 +174,9 @@ public SearchRequest(StreamInput in) throws IOException { } //TODO update version after backport if (in.getVersion().onOrAfter(Version.V_7_0_0)) { - clusterAlias = in.readOptionalString(); + localClusterAlias = in.readOptionalString(); } else { - clusterAlias = null; + localClusterAlias = null; } } @@ -203,7 +203,7 @@ public void writeTo(StreamOutput out) throws IOException { } //TODO update version after backport if (out.getVersion().onOrAfter(Version.V_7_0_0)) { - out.writeOptionalString(clusterAlias); + out.writeOptionalString(localClusterAlias); } } @@ -235,12 +235,12 @@ public ActionRequestValidationException validate() { /** * Returns the alias of the cluster that this search request is being executed on. A non-null value indicates that this search request - * is being executed as part of a locally reduced cross-cluster search request. The cluster alias is used to provide the cluster alias - * to prefix returned index names with. + * is being executed as part of a locally reduced cross-cluster search request. The cluster alias is used to prefix index names + * returned as part of search hits with the alias of the cluster they came from. */ @Nullable - String getClusterAlias() { - return clusterAlias; + String getLocalClusterAlias() { + return localClusterAlias; } /** @@ -564,14 +564,14 @@ public boolean equals(Object o) { Objects.equals(preFilterShardSize, that.preFilterShardSize) && Objects.equals(indicesOptions, that.indicesOptions) && Objects.equals(allowPartialSearchResults, that.allowPartialSearchResults) && - Objects.equals(clusterAlias, that.clusterAlias); + Objects.equals(localClusterAlias, that.localClusterAlias); } @Override public int hashCode() { return Objects.hash(searchType, Arrays.hashCode(indices), routing, preference, source, requestCache, scroll, Arrays.hashCode(types), indicesOptions, batchedReduceSize, maxConcurrentShardRequests, preFilterShardSize, - allowPartialSearchResults, clusterAlias); + allowPartialSearchResults, localClusterAlias); } @Override @@ -589,7 +589,7 @@ public String toString() { ", batchedReduceSize=" + batchedReduceSize + ", preFilterShardSize=" + preFilterShardSize + ", allowPartialSearchResults=" + allowPartialSearchResults + - ", clusterAlias=" + clusterAlias + + ", localClusterAlias=" + localClusterAlias + ", source=" + source + '}'; } } diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index b866dc6764aed..9df930544e624 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -313,7 +313,7 @@ private void executeSearch(SearchTask task, SearchTimeProvider timeProvider, Sea GroupShardsIterator localShardsIterator = clusterService.operationRouting().searchShards(clusterState, concreteIndices, routingMap, searchRequest.preference(), searchService.getResponseCollectorService(), nodeSearchCounts); GroupShardsIterator shardIterators = mergeShardsIterators(localShardsIterator, localIndices, - searchRequest.getClusterAlias(), remoteShardIterators); + searchRequest.getLocalClusterAlias(), remoteShardIterators); failIfOverShardCountLimit(clusterService, shardIterators.size()); @@ -340,7 +340,7 @@ private void executeSearch(SearchTask task, SearchTimeProvider timeProvider, Sea } final DiscoveryNodes nodes = clusterState.nodes(); - BiFunction connectionLookup = buildConnectionLookup(searchRequest.getClusterAlias(), + BiFunction connectionLookup = buildConnectionLookup(searchRequest.getLocalClusterAlias(), nodes::get, remoteConnections, searchTransportService::getConnection); boolean preFilterSearchShards = shouldPreFilterSearchShards(searchRequest, shardIterators); searchAsyncAction(task, searchRequest, shardIterators, timeProvider, connectionLookup, clusterState.version(), diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java index c640390708b6a..719a14491ae1d 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java @@ -68,9 +68,9 @@ public void testClusterAliasSerialization() throws IOException { SearchRequest deserializedRequest = copyWriteable(searchRequest, namedWriteableRegistry, SearchRequest::new, version); //TODO update version after backport if (version.before(Version.V_7_0_0)) { - assertNull(deserializedRequest.getClusterAlias()); + assertNull(deserializedRequest.getLocalClusterAlias()); } else { - assertEquals(searchRequest.getClusterAlias(), deserializedRequest.getClusterAlias()); + assertEquals(searchRequest.getLocalClusterAlias(), deserializedRequest.getLocalClusterAlias()); } } @@ -80,7 +80,7 @@ public void testReadFromPre7_0_0() throws IOException { try (StreamInput in = StreamInput.wrap(Base64.getDecoder().decode(msg))) { SearchRequest searchRequest = new SearchRequest(in); assertArrayEquals(new String[]{"index"}, searchRequest.indices()); - assertNull(searchRequest.getClusterAlias()); + assertNull(searchRequest.getLocalClusterAlias()); } }