From ecf9175dac907e011d17a2ad65015b4c22c1fe78 Mon Sep 17 00:00:00 2001 From: Michael Peterson Date: Fri, 26 May 2023 16:16:05 -0400 Subject: [PATCH 1/8] Profile API should show node details as well as shard details SearchProfileResults has additional fields for XContent output: node_id, cluster, index, shard_id. It parses the existing composite ID using the new parseProfileShardId method, which reverses the SeachShardTarget.toString method. No new information is added here, merely the splitting out of the four pieces of information in the profile shards "composite" id that is created by the SeachShardTarget.toString method. Profile/shards output now has the form: "profile": { "shards": [ { "id": "[2m7SW9oIRrirdrwirM1mwQ][blogs][0]", "node_id": "2m7SW9oIRrirdrwirM1mwQ", "shard_id": "0", "index": "blogs", "cluster": "(local)", "searches": [ ... ] ... }, { "id": "[UngEVXTBQL-7w5j_tftGAQ][remote1:blogs][2]", "node_id": "UngEVXTBQL-7w5j_tftGAQ", "shard_id": "2", "index": "blogs", "cluster": "remote1", "searches": [ ... ] ... where the latter is on a remote cluster and you can see that as the prefix on the index name. Partially addresses #25896 --- docs/reference/search/profile.asciidoc | 109 ++++++++++++------ .../search/SearchShardTarget.java | 6 +- .../search/profile/SearchProfileResults.java | 78 ++++++++++++- .../profile/SearchProfileResultsBuilder.java | 1 + .../profile/SearchProfileShardResult.java | 2 + .../profile/SearchProfileResultsTests.java | 26 +++++ 6 files changed, 183 insertions(+), 39 deletions(-) diff --git a/docs/reference/search/profile.asciidoc b/docs/reference/search/profile.asciidoc index 15d2da5dd59d7..0256dad1f4c59 100644 --- a/docs/reference/search/profile.asciidoc +++ b/docs/reference/search/profile.asciidoc @@ -71,7 +71,11 @@ The API returns the following result: "profile": { "shards": [ { - "id": "[2aE02wS1R8q_QFnYu6vDVQ][my-index-000001][0]", + "id": "[q2aE02wS1R8qQFnYu6vDVQ][my-index-000001][0]", + "node_id": "q2aE02wS1R8qQFnYu6vDVQ", + "shard_id": 0, + "index": "my-index-000001", + "cluster": "(local)", "searches": [ { "query": [ @@ -221,7 +225,9 @@ The API returns the following result: // TESTRESPONSE[s/"took": 25/"took": $body.took/] // TESTRESPONSE[s/"hits": \[...\]/"hits": $body.$_path/] // TESTRESPONSE[s/(?<=[" ])\d+(\.\d+)?/$body.$_path/] -// TESTRESPONSE[s/\[2aE02wS1R8q_QFnYu6vDVQ\]\[my-index-000001\]\[0\]/$body.$_path/] +// TESTRESPONSE[s/"id": "\[q2aE02wS1R8qQFnYu6vDVQ\]\[my-index-000001\]\[0\]"/"id": $body.profile.shards.0.id/] +// TESTRESPONSE[s/"node_id": "q2aE02wS1R8qQFnYu6vDVQ",/"node_id": "$body.profile.shards.0.node_id",/] + <1> Search results are returned, but were omitted here for brevity. @@ -237,16 +243,20 @@ The overall structure of the profile response is as follows: "profile": { "shards": [ { - "id": "[2aE02wS1R8q_QFnYu6vDVQ][my-index-000001][0]", <1> + "id": "[q2aE02wS1R8qQFnYu6vDVQ][my-index-000001][0]", <1> + "node_id": "q2aE02wS1R8qQFnYu6vDVQ", + "shard_id": 0, + "index": "my-index-000001", + "cluster": "(local)", <2> "searches": [ { - "query": [...], <2> - "rewrite_time": 51443, <3> - "collector": [...] <4> + "query": [...], <3> + "rewrite_time": 51443, <4> + "collector": [...] <5> } ], - "aggregations": [...], <5> - "fetch": {...} <6> + "aggregations": [...], <6> + "fetch": {...} <7> } ] } @@ -254,23 +264,32 @@ The overall structure of the profile response is as follows: -------------------------------------------------- // TESTRESPONSE[s/"profile": /"took": $body.took, "timed_out": $body.timed_out, "_shards": $body._shards, "hits": $body.hits, "profile": /] // TESTRESPONSE[s/(?<=[" ])\d+(\.\d+)?/$body.$_path/] -// TESTRESPONSE[s/\[2aE02wS1R8q_QFnYu6vDVQ\]\[my-index-000001\]\[0\]/$body.$_path/] +// TESTRESPONSE[s/"id": "\[q2aE02wS1R8qQFnYu6vDVQ\]\[my-index-000001\]\[0\]"/"id": $body.profile.shards.0.id/] +// TESTRESPONSE[s/"node_id": "q2aE02wS1R8qQFnYu6vDVQ",/"node_id": "$body.profile.shards.0.node_id",/] // TESTRESPONSE[s/"query": \[...\]/"query": $body.$_path/] // TESTRESPONSE[s/"collector": \[...\]/"collector": $body.$_path/] // TESTRESPONSE[s/"aggregations": \[...\]/"aggregations": []/] // TESTRESPONSE[s/"fetch": \{...\}/"fetch": $body.$_path/] <1> A profile is returned for each shard that participated in the response, and -is identified by a unique ID. -<2> Query timings and other debugging information. -<3> The cumulative rewrite time. -<4> Names and invocation timings for each collector. -<5> Aggregation timings, invocation counts, and debug information. -<6> Fetch timing and debug information. +is identified by a unique composite ID. The parts of that ID are split out into +the "node_id", "shard_id", "index" and "cluster" fields below. +<2> If the query was run on the local cluster, the cluster name is left out of the +composite id and is marked "(local)" here. For a profile running on a remote_cluster +using cross-cluster search, the "id" value would be something like +`[q2aE02wS1R8qQFnYu6vDVQ][remote1:my-index-000001][0]` and the "cluster" +value would be `remote1`. +<3> Query timings and other debugging information. +<4> The cumulative rewrite time. +<5> Names and invocation timings for each collector. +<6> Aggregation timings, invocation counts, and debug information. +<7> Fetch timing and debug information. Because a search request may be executed against one or more shards in an index, and a search may cover one or more indices, the top level element in the profile response is an array of `shard` objects. Each shard object lists its `id` which uniquely identifies the shard. The ID's format is +`[nodeID][clusterName:indexName][shardID]`. If the search is run against the +local cluster then the clusterName is not added and the format is `[nodeID][indexName][shardID]`. The profile itself may consist of one or more "searches", where a search is a @@ -343,7 +362,7 @@ Using our previous `match` query example, let's analyze the `query` section: } ] -------------------------------------------------- -// TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ {\n"id": "$body.$_path",\n"searches": [{\n/] +// TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ {\n"id": "$body.profile.shards.0.id",\n"node_id": "$body.profile.shards.0.node_id",\n"shard_id": $body.profile.shards.0.shard_id,\n"index": "$body.profile.shards.0.index",\n"cluster": "(local)",\n"searches": [{\n/] // TESTRESPONSE[s/]$/],"rewrite_time": $body.$_path, "collector": $body.$_path}], "aggregations": [], "fetch": $body.$_path}]}}/] // TESTRESPONSE[s/(?<=[" ])\d+(\.\d+)?/$body.$_path/] // TESTRESPONSE[s/"breakdown": \{...\}/"breakdown": $body.$_path/] @@ -367,6 +386,7 @@ search"), our BooleanQuery holds two children TermQueries. They have identical information (type, time, breakdown, etc). Children are allowed to have their own children. + ===== Timing Breakdown The `breakdown` component lists detailed timing statistics about low-level @@ -397,7 +417,7 @@ Lucene execution: "count_weight_count": 0 } -------------------------------------------------- -// TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ {\n"id": "$body.$_path",\n"searches": [{\n"query": [{\n"type": "BooleanQuery",\n"description": "message:get message:search",\n"time_in_nanos": $body.$_path,/] +// TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ {\n"id": "$body.profile.shards.0.id",\n"node_id": "$body.profile.shards.0.node_id",\n"shard_id": $body.profile.shards.0.shard_id,\n"index": "$body.profile.shards.0.index",\n"cluster": "(local)",\n"searches": [{\n"query": [{\n"type": "BooleanQuery",\n"description": "message:get message:search",\n"time_in_nanos": $body.$_path,/] // TESTRESPONSE[s/}$/},\n"children": $body.$_path}],\n"rewrite_time": $body.$_path, "collector": $body.$_path}], "aggregations": [], "fetch": $body.$_path}]}}/] // TESTRESPONSE[s/(?<=[" ])\d+(\.\d+)?/$body.$_path/] @@ -416,15 +436,15 @@ The meaning of the stats are as follows: [horizontal] `create_weight`:: - A Query in Lucene must be capable of reuse across multiple IndexSearchers (think of it as the engine that - executes a search against a specific Lucene Index). This puts Lucene in a tricky spot, since many queries - need to accumulate temporary state/statistics associated with the index it is being used against, but the - Query contract mandates that it must be immutable. - {empty} + - {empty} + - To get around this, Lucene asks each query to generate a Weight object which acts as a temporary context - object to hold state associated with this particular (IndexSearcher, Query) tuple. The `weight` metric - shows how long this process takes +A Query in Lucene must be capable of reuse across multiple IndexSearchers (think of it as the engine that +executes a search against a specific Lucene Index). This puts Lucene in a tricky spot, since many queries +need to accumulate temporary state/statistics associated with the index it is being used against, but the +Query contract mandates that it must be immutable. +{empty} + +{empty} + +To get around this, Lucene asks each query to generate a Weight object which acts as a temporary context +object to hold state associated with this particular (IndexSearcher, Query) tuple. The `weight` metric +shows how long this process takes `build_scorer`:: @@ -472,9 +492,10 @@ The meaning of the stats are as follows: This records the time taken to score a particular document via its Scorer `*_count`:: - Records the number of invocations of the particular method. For example, `"next_doc_count": 2,` - means the `nextDoc()` method was called on two different documents. This can be used to help judge - how selective queries are, by comparing counts between different query components. +Records the number of invocations of the particular method. For example, `"next_doc_count": 2,` +means the `nextDoc()` method was called on two different documents. This can be used to help judge +how selective queries are, by comparing counts between different query components. + [[collectors-section]] @@ -498,7 +519,7 @@ Looking at the previous example: } ] -------------------------------------------------- -// TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ {\n"id": "$body.$_path",\n"searches": [{\n"query": $body.$_path,\n"rewrite_time": $body.$_path,/] +// TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ {\n"id": "$body.profile.shards.0.id",\n"node_id": "$body.profile.shards.0.node_id",\n"shard_id": $body.profile.shards.0.shard_id,\n"index": "$body.profile.shards.0.index",\n"cluster": "(local)",\n"searches": [{\n"query": $body.$_path,\n"rewrite_time": $body.$_path,/] // TESTRESPONSE[s/]$/]}], "aggregations": [], "fetch": $body.$_path}]}}/] // TESTRESPONSE[s/(?<=[" ])\d+(\.\d+)?/$body.$_path/] @@ -522,8 +543,8 @@ For reference, the various collector reasons are: [horizontal] `search_sorted`:: - A collector that scores and sorts documents. This is the most common collector and will be seen in most - simple searches +A collector that scores and sorts documents. This is the most common collector and will be seen in most +simple searches `search_count`:: @@ -563,6 +584,7 @@ For reference, the various collector reasons are: match_all query (which you will see added to the Query section) to collect your entire dataset + [[rewrite-section]] ===== `rewrite` Section @@ -620,7 +642,7 @@ GET /my-index-000001/_search } -------------------------------------------------- // TEST[setup:my_index] -// TEST[s/_search/_search\?filter_path=profile.shards.id,profile.shards.searches,profile.shards.aggregations,profile.shards.fetch/] +// TEST[s/_search/_search\?filter_path=profile.shards.id,profile.shards.node_id,profile.shards.shard_id,profile.shards.index,profile.shards.cluster,profile.shards.searches,profile.shards.aggregations,profile.shards.fetch/] This example has: @@ -640,7 +662,11 @@ The API returns the following result: "profile": { "shards": [ { - "id": "[P6-vulHtQRWuD4YnubWb7A][my-index-000001][0]", + "id": "[P6xvulHtQRWuD4YnubWb7A][my-index-000001][0]", + "node_id": "P6xvulHtQRWuD4YnubWb7A", + "shard_id": 0, + "index": "my-index-000001", + "cluster": "(local)", "searches": [ { "query": [ @@ -739,7 +765,9 @@ The API returns the following result: // TESTRESPONSE[s/"fetch": \{\.\.\.\}/"fetch": $body.$_path/] // TESTRESPONSE[s/\.\.\.//] // TESTRESPONSE[s/(?<=[" ])\d+(\.\d+)?/$body.$_path/] -// TESTRESPONSE[s/"id": "\[P6-vulHtQRWuD4YnubWb7A\]\[my-index-000001\]\[0\]"/"id": $body.profile.shards.0.id/] +// TESTRESPONSE[s/"id": "\[P6xvulHtQRWuD4YnubWb7A\]\[my-index-000001\]\[0\]"/"id": $body.profile.shards.0.id/] +// TESTRESPONSE[s/"node_id": "P6xvulHtQRWuD4YnubWb7A",/"node_id": "$body.profile.shards.0.node_id",/] + <1> The `"aggregations"` portion has been omitted because it will be covered in the next section. @@ -754,6 +782,8 @@ CancellableCollector wraps a MultiCollector which also wraps a FilteredCollector to execute the post_filter (and in turn wraps the normal scoring SimpleCollector), a BucketCollector to run all scoped aggregations. + + ===== Understanding MultiTermQuery output A special note needs to be made about the `MultiTermQuery` class of queries. @@ -940,6 +970,7 @@ hack on aggregations but that we don't expect to be otherwise useful. They can vary wildly between versions, aggregations, and aggregation execution strategies. + ===== Timing Breakdown The `breakdown` component lists detailed statistics about low-level execution: @@ -1094,6 +1125,7 @@ multiple shards. Both of these cases can be profiled by setting `profile` to `true` as part of the search request. + [[profiling-dfs-statistics]] ====== Profiling DFS Statistics @@ -1176,14 +1208,17 @@ One of the `dfs` sections for a shard looks like the following: } } -------------------------------------------------- -// TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ "$body.$_path", {\n"id": "$body.$_path",\n/] +// TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ "$body.$_path", {\n"id": "$body.$_path",\n"node_id": "$body.$_path",\n"shard_id": "$body.$_path",\n"index": "$body.$_path",\n"cluster": "$body.$_path",\n/] // TESTRESPONSE[s/}$/}, "aggregations": [], "searches": $body.$_path}]}}/] // TESTRESPONSE[s/(\-)?[0-9]+/ $body.$_path/] + In the `dfs.statistics` portion of this response we see a `time_in_nanos` which is the total time it took to collect term statistics for this shard along with a further breakdown of the individual parts. + + [[profiling-knn-search]] ====== Profiling kNN Search @@ -1290,7 +1325,7 @@ One of the `dfs.knn` sections for a shard looks like the following: } ] } -------------------------------------------------- -// TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ {\n"id": "$body.$_path",\n/] +// TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ {\n"id": "$body.$_path",\n"node_id": "$body.$_path",\n"shard_id": "$body.$_path",\n"index": "$body.$_path",\n"cluster": "$body.$_path",\n/] // TESTRESPONSE[s/}$/}, "aggregations": [], "searches": $body.$_path, "fetch": $body.$_path}]}}/] // TESTRESPONSE[s/ (\-)?[0-9]+/ $body.$_path/] // TESTRESPONSE[s/"dfs" : \{/"dfs" : {"statistics": $body.$_path,/] diff --git a/server/src/main/java/org/elasticsearch/search/SearchShardTarget.java b/server/src/main/java/org/elasticsearch/search/SearchShardTarget.java index 00cabc7c86172..ef956fcd2251c 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchShardTarget.java +++ b/server/src/main/java/org/elasticsearch/search/SearchShardTarget.java @@ -23,7 +23,6 @@ * The target that the search request was executed on. */ public final class SearchShardTarget implements Writeable, Comparable { - private final Text nodeId; private final ShardId shardId; private final String clusterAlias; @@ -113,6 +112,11 @@ public int hashCode() { return Objects.hash(nodeId, shardId, clusterAlias); } + /** + * NOTE: this representation is used as the "id" for shards for the REST response + * when query profiling is requested. So changing this formulation may break + * systems that rely on the format, including the parser in SearchProfileResults. + */ @Override public String toString() { String shardToString = "[" diff --git a/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResults.java b/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResults.java index c993edb5b1dd5..5ef5fe270ffc0 100644 --- a/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResults.java +++ b/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResults.java @@ -8,11 +8,14 @@ package org.elasticsearch.search.profile; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.TransportVersion; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.core.Nullable; import org.elasticsearch.search.profile.aggregation.AggregationProfileShardResult; import org.elasticsearch.search.profile.query.QueryProfileShardResult; import org.elasticsearch.xcontent.ToXContentFragment; @@ -26,6 +29,8 @@ import java.util.List; import java.util.Map; import java.util.TreeSet; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; @@ -34,10 +39,16 @@ */ public final class SearchProfileResults implements Writeable, ToXContentFragment { + private static final Logger logger = LogManager.getLogger(SearchProfileResults.class); private static final String ID_FIELD = "id"; + private static final String NODE_ID_FIELD = "node_id"; + private static final String CLUSTER_FIELD = "cluster"; + private static final String INDEX_NAME_FIELD = "index"; + private static final String SHARD_ID_FIELD = "shard_id"; private static final String SHARDS_FIELD = "shards"; public static final String PROFILE_FIELD = "profile"; + /// MP: key to this map is [nodeId] [remote-index-name] [shardId] created from the SearchShardTarget.toString method private Map shardResults; public SearchProfileResults(Map shardResults) { @@ -76,9 +87,24 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws for (String key : sortedKeys) { builder.startObject(); builder.field(ID_FIELD, key); - shardResults.get(key).toXContent(builder, params); + + ShardProfileId shardProfileId = parseProfileShardId(key); + if (shardProfileId != null) { + builder.field(NODE_ID_FIELD, shardProfileId.nodeId()); + builder.field(SHARD_ID_FIELD, shardProfileId.shardId()); + builder.field(INDEX_NAME_FIELD, shardProfileId.indexName()); + String cluster = shardProfileId.clusterName(); + if (cluster == null) { + cluster = "(local)"; + } + builder.field(CLUSTER_FIELD, cluster); + } + + SearchProfileShardResult shardResult = shardResults.get(key); + shardResult.toXContent(builder, params); builder.endObject(); } + builder.endArray().endObject(); return builder; } @@ -122,6 +148,7 @@ public static SearchProfileResults fromXContent(XContentParser parser) throws IO return new SearchProfileResults(profileResults); } + /// MP: what is the key to this map? private static void parseProfileResultsEntry(XContentParser parser, Map searchProfileResults) throws IOException { XContentParser.Token token = parser.currentToken(); @@ -170,4 +197,53 @@ private static void parseProfileResultsEntry(XContentParser parser, Map + * One of two expected patterns is accepted: + *

+ * 1) [nodeId][indexName][shardId] + * example: [2m7SW9oIRrirdrwirM1mwQ][blogs][1] + *

+ * 2) [nodeId][clusterName:indexName][shardId] + * example: [UngEVXTBQL-7w5j_tftGAQ][remote1:blogs][0] + * + * @param compositeId see above for accepted formats + * @return ShardProfileId with parsed components or null if the compositeId has an unsupported format + */ + static ShardProfileId parseProfileShardId(String compositeId) { + if (Strings.isNullOrEmpty(compositeId)) { + return null; + } + Pattern r = Pattern.compile("\\[([^]]+)\\]\\[([^]]+)\\]\\[(\\d+)\\]"); + Matcher m = r.matcher(compositeId); + if (m.find()) { + String nodeId = m.group(1); + String indexName = m.group(2); + int shardId = Integer.parseInt(m.group(3)); + String cluster = null; + if (indexName.contains(":")) { + // index names and cluster names cannot contain a ':', so this split should be accurate + String[] tokens = indexName.split(":", 2); + cluster = tokens[0]; + indexName = tokens[1]; + } + return new ShardProfileId(nodeId, indexName, shardId, cluster); + } else { + logger.warn("Unable to match input against expected pattern of [nodeId][indexName][shardId]. Input: {}", compositeId); + return null; + } + } } diff --git a/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResultsBuilder.java b/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResultsBuilder.java index 17c12af348571..a91dcf509ab88 100644 --- a/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResultsBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResultsBuilder.java @@ -34,6 +34,7 @@ public SearchProfileResults build(Collection fetchR Map mergedShardResults = Maps.newMapWithExpectedSize(queryPhaseResults.size()); for (SearchPhaseResult r : fetchResults) { FetchSearchResult fr = r.fetchResult(); + /// MP: THIS IS WHERE THE _key_ for the Map is created String key = fr.getSearchShardTarget().toString(); SearchProfileQueryPhaseResult queryPhase = queryPhaseResults.get(key); if (queryPhase == null) { diff --git a/server/src/main/java/org/elasticsearch/search/profile/SearchProfileShardResult.java b/server/src/main/java/org/elasticsearch/search/profile/SearchProfileShardResult.java index e091a163d5ae4..c15229fb9e98b 100644 --- a/server/src/main/java/org/elasticsearch/search/profile/SearchProfileShardResult.java +++ b/server/src/main/java/org/elasticsearch/search/profile/SearchProfileShardResult.java @@ -30,6 +30,8 @@ public class SearchProfileShardResult implements Writeable, ToXContentFragment { private final SearchProfileQueryPhaseResult queryPhase; private final ProfileResult fetchPhase; + /// MP: IDEA add SearchShardTarget as instance field - is this possible? + public SearchProfileShardResult(SearchProfileQueryPhaseResult queryPhase, @Nullable ProfileResult fetch) { this.queryPhase = queryPhase; this.fetchPhase = fetch; diff --git a/server/src/test/java/org/elasticsearch/search/profile/SearchProfileResultsTests.java b/server/src/test/java/org/elasticsearch/search/profile/SearchProfileResultsTests.java index 159ccccf74306..0fbc0fa01dc04 100644 --- a/server/src/test/java/org/elasticsearch/search/profile/SearchProfileResultsTests.java +++ b/server/src/test/java/org/elasticsearch/search/profile/SearchProfileResultsTests.java @@ -32,6 +32,32 @@ public static SearchProfileResults createTestItem() { return new SearchProfileResults(shards); } + public void testParseProfileShardId() { + record TestPattern(String id, SearchProfileResults.ShardProfileId expected) {} + + TestPattern[] cases = new TestPattern[] { + new TestPattern(null, null), + new TestPattern("", null), + new TestPattern("[][][]", null), + new TestPattern("chsk8ad", null), + new TestPattern("[UngEVXTBQL-7w5j_tftGAQ][remote1:blogs]", null), // shardId is missing + new TestPattern("[UngEVXTBQL-7w5j_tftGAQ][remote1:blogs][xyz]", null), // shardId must be integer + + new TestPattern( + "[UngEVXTBQL-7w5j_tftGAQ][remote1:blogs][0]", + new SearchProfileResults.ShardProfileId("UngEVXTBQL-7w5j_tftGAQ", "blogs", 0, "remote1") + ), + new TestPattern( + "[2m7SW9oIRrirdrwirM1mwQ][.ds-filebeat-8.3.2-2023.05.04-000420][303]", + new SearchProfileResults.ShardProfileId("2m7SW9oIRrirdrwirM1mwQ", ".ds-filebeat-8.3.2-2023.05.04-000420", 303, null) + ), + new TestPattern("[_na_][:indexName_1][303]", new SearchProfileResults.ShardProfileId("_na_", "indexName_1", 303, "")) }; + + for (TestPattern testPattern : cases) { + assertEquals(testPattern.expected, SearchProfileResults.parseProfileShardId(testPattern.id)); + } + } + @Override protected SearchProfileResults createTestInstance() { return createTestItem(); From 9885849de8d97de910dc5621d3d3ca911458d001 Mon Sep 17 00:00:00 2001 From: Michael Peterson Date: Fri, 9 Jun 2023 17:20:47 -0400 Subject: [PATCH 2/8] Properly updated profile.asciidoc with all embedded test snippets passing --- docs/reference/search/profile.asciidoc | 43 +++++++++----------------- 1 file changed, 15 insertions(+), 28 deletions(-) diff --git a/docs/reference/search/profile.asciidoc b/docs/reference/search/profile.asciidoc index 0256dad1f4c59..26d55350da9ec 100644 --- a/docs/reference/search/profile.asciidoc +++ b/docs/reference/search/profile.asciidoc @@ -228,7 +228,6 @@ The API returns the following result: // TESTRESPONSE[s/"id": "\[q2aE02wS1R8qQFnYu6vDVQ\]\[my-index-000001\]\[0\]"/"id": $body.profile.shards.0.id/] // TESTRESPONSE[s/"node_id": "q2aE02wS1R8qQFnYu6vDVQ",/"node_id": "$body.profile.shards.0.node_id",/] - <1> Search results are returned, but were omitted here for brevity. Even for a simple query, the response is relatively complicated. Let's break it @@ -271,8 +270,7 @@ The overall structure of the profile response is as follows: // TESTRESPONSE[s/"aggregations": \[...\]/"aggregations": []/] // TESTRESPONSE[s/"fetch": \{...\}/"fetch": $body.$_path/] <1> A profile is returned for each shard that participated in the response, and -is identified by a unique composite ID. The parts of that ID are split out into -the "node_id", "shard_id", "index" and "cluster" fields below. +is identified by a unique ID. <2> If the query was run on the local cluster, the cluster name is left out of the composite id and is marked "(local)" here. For a profile running on a remote_cluster using cross-cluster search, the "id" value would be something like @@ -386,7 +384,6 @@ search"), our BooleanQuery holds two children TermQueries. They have identical information (type, time, breakdown, etc). Children are allowed to have their own children. - ===== Timing Breakdown The `breakdown` component lists detailed timing statistics about low-level @@ -436,15 +433,15 @@ The meaning of the stats are as follows: [horizontal] `create_weight`:: -A Query in Lucene must be capable of reuse across multiple IndexSearchers (think of it as the engine that -executes a search against a specific Lucene Index). This puts Lucene in a tricky spot, since many queries -need to accumulate temporary state/statistics associated with the index it is being used against, but the -Query contract mandates that it must be immutable. -{empty} + -{empty} + -To get around this, Lucene asks each query to generate a Weight object which acts as a temporary context -object to hold state associated with this particular (IndexSearcher, Query) tuple. The `weight` metric -shows how long this process takes + A Query in Lucene must be capable of reuse across multiple IndexSearchers (think of it as the engine that + executes a search against a specific Lucene Index). This puts Lucene in a tricky spot, since many queries + need to accumulate temporary state/statistics associated with the index it is being used against, but the + Query contract mandates that it must be immutable. + {empty} + + {empty} + + To get around this, Lucene asks each query to generate a Weight object which acts as a temporary context + object to hold state associated with this particular (IndexSearcher, Query) tuple. The `weight` metric + shows how long this process takes `build_scorer`:: @@ -492,10 +489,9 @@ shows how long this process takes This records the time taken to score a particular document via its Scorer `*_count`:: -Records the number of invocations of the particular method. For example, `"next_doc_count": 2,` -means the `nextDoc()` method was called on two different documents. This can be used to help judge -how selective queries are, by comparing counts between different query components. - + Records the number of invocations of the particular method. For example, `"next_doc_count": 2,` + means the `nextDoc()` method was called on two different documents. This can be used to help judge + how selective queries are, by comparing counts between different query components. [[collectors-section]] @@ -543,8 +539,8 @@ For reference, the various collector reasons are: [horizontal] `search_sorted`:: -A collector that scores and sorts documents. This is the most common collector and will be seen in most -simple searches + A collector that scores and sorts documents. This is the most common collector and will be seen in most + simple searches `search_count`:: @@ -584,7 +580,6 @@ simple searches match_all query (which you will see added to the Query section) to collect your entire dataset - [[rewrite-section]] ===== `rewrite` Section @@ -767,7 +762,6 @@ The API returns the following result: // TESTRESPONSE[s/(?<=[" ])\d+(\.\d+)?/$body.$_path/] // TESTRESPONSE[s/"id": "\[P6xvulHtQRWuD4YnubWb7A\]\[my-index-000001\]\[0\]"/"id": $body.profile.shards.0.id/] // TESTRESPONSE[s/"node_id": "P6xvulHtQRWuD4YnubWb7A",/"node_id": "$body.profile.shards.0.node_id",/] - <1> The `"aggregations"` portion has been omitted because it will be covered in the next section. @@ -782,8 +776,6 @@ CancellableCollector wraps a MultiCollector which also wraps a FilteredCollector to execute the post_filter (and in turn wraps the normal scoring SimpleCollector), a BucketCollector to run all scoped aggregations. - - ===== Understanding MultiTermQuery output A special note needs to be made about the `MultiTermQuery` class of queries. @@ -970,7 +962,6 @@ hack on aggregations but that we don't expect to be otherwise useful. They can vary wildly between versions, aggregations, and aggregation execution strategies. - ===== Timing Breakdown The `breakdown` component lists detailed statistics about low-level execution: @@ -1125,7 +1116,6 @@ multiple shards. Both of these cases can be profiled by setting `profile` to `true` as part of the search request. - [[profiling-dfs-statistics]] ====== Profiling DFS Statistics @@ -1212,13 +1202,10 @@ One of the `dfs` sections for a shard looks like the following: // TESTRESPONSE[s/}$/}, "aggregations": [], "searches": $body.$_path}]}}/] // TESTRESPONSE[s/(\-)?[0-9]+/ $body.$_path/] - In the `dfs.statistics` portion of this response we see a `time_in_nanos` which is the total time it took to collect term statistics for this shard along with a further breakdown of the individual parts. - - [[profiling-knn-search]] ====== Profiling kNN Search From fe9b1f77b1db3161bb30da3f41e3d5443e825a8c Mon Sep 17 00:00:00 2001 From: Michael Peterson Date: Fri, 9 Jun 2023 17:28:31 -0400 Subject: [PATCH 3/8] code comment cleanup --- .../org/elasticsearch/search/profile/SearchProfileResults.java | 3 +-- .../search/profile/SearchProfileResultsBuilder.java | 1 - .../elasticsearch/search/profile/SearchProfileShardResult.java | 2 -- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResults.java b/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResults.java index 5ef5fe270ffc0..0fbce6ae1ef4d 100644 --- a/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResults.java +++ b/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResults.java @@ -48,7 +48,7 @@ public final class SearchProfileResults implements Writeable, ToXContentFragment private static final String SHARDS_FIELD = "shards"; public static final String PROFILE_FIELD = "profile"; - /// MP: key to this map is [nodeId] [remote-index-name] [shardId] created from the SearchShardTarget.toString method + // map key is the composite "id" of form [nodeId][(clusterName:)indexName][shardId] created from SearchShardTarget.toString private Map shardResults; public SearchProfileResults(Map shardResults) { @@ -148,7 +148,6 @@ public static SearchProfileResults fromXContent(XContentParser parser) throws IO return new SearchProfileResults(profileResults); } - /// MP: what is the key to this map? private static void parseProfileResultsEntry(XContentParser parser, Map searchProfileResults) throws IOException { XContentParser.Token token = parser.currentToken(); diff --git a/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResultsBuilder.java b/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResultsBuilder.java index a91dcf509ab88..17c12af348571 100644 --- a/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResultsBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResultsBuilder.java @@ -34,7 +34,6 @@ public SearchProfileResults build(Collection fetchR Map mergedShardResults = Maps.newMapWithExpectedSize(queryPhaseResults.size()); for (SearchPhaseResult r : fetchResults) { FetchSearchResult fr = r.fetchResult(); - /// MP: THIS IS WHERE THE _key_ for the Map is created String key = fr.getSearchShardTarget().toString(); SearchProfileQueryPhaseResult queryPhase = queryPhaseResults.get(key); if (queryPhase == null) { diff --git a/server/src/main/java/org/elasticsearch/search/profile/SearchProfileShardResult.java b/server/src/main/java/org/elasticsearch/search/profile/SearchProfileShardResult.java index c15229fb9e98b..e091a163d5ae4 100644 --- a/server/src/main/java/org/elasticsearch/search/profile/SearchProfileShardResult.java +++ b/server/src/main/java/org/elasticsearch/search/profile/SearchProfileShardResult.java @@ -30,8 +30,6 @@ public class SearchProfileShardResult implements Writeable, ToXContentFragment { private final SearchProfileQueryPhaseResult queryPhase; private final ProfileResult fetchPhase; - /// MP: IDEA add SearchShardTarget as instance field - is this possible? - public SearchProfileShardResult(SearchProfileQueryPhaseResult queryPhase, @Nullable ProfileResult fetch) { this.queryPhase = queryPhase; this.fetchPhase = fetch; From ab51c501a5996feec7bd7d33e852706b6429bfd0 Mon Sep 17 00:00:00 2001 From: Michael Peterson Date: Mon, 12 Jun 2023 09:39:03 -0400 Subject: [PATCH 4/8] Fixed javadoc for record ShardProfileId --- .../elasticsearch/search/profile/SearchProfileResults.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResults.java b/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResults.java index 0fbce6ae1ef4d..739e2a8d91c40 100644 --- a/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResults.java +++ b/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResults.java @@ -199,10 +199,10 @@ private static void parseProfileResultsEntry(XContentParser parser, Map Date: Fri, 16 Jun 2023 15:56:45 -0400 Subject: [PATCH 5/8] PR feedback Added asserts to the SearchProfileResults.parseCompositeProfileShardId method to assert against invalid inputs Added yamlRestTest for the new fields in the profile response. Updated the existing unit test to use more randomization of inputs and to use the SearchShardTarget.toString method to generate the compositeId (which is what happens in production) --- .../rest-api-spec/test/search/370_profile.yml | 27 ++++++++ .../search/profile/SearchProfileResults.java | 10 +-- .../profile/SearchProfileResultsTests.java | 63 ++++++++++++++----- 3 files changed, 79 insertions(+), 21 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml index c0b754c02275f..ea40cebcc25eb 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml @@ -401,3 +401,30 @@ dfs profile for search with dfs_query_then_fetch: - match: { hits.total.value: 0 } - is_false: profile.shards.0.dfs +--- +composite_id_parsed: + - skip: + version: " - 8.8.99" + reason: node_id, shard_id, index and cluster fields added in 8.9.0 + features: "contains" + + - do: + search: + index: test + body: + _source: false + profile: true + fields: [keyword] + - set: { profile.shards.0.id: composite_id, + profile.shards.0.node_id: first_node_id, + profile.shards.0.index: first_index + } + + - is_true: profile.shards.0.id + - is_true: profile.shards.0.node_id + - length: { profile.shards.0.node_id: 22 } + - gte: { profile.shards.0.shard_id: 0 } + - match: { profile.shards.0.index: "test" } + - match: { profile.shards.0.cluster: "(local)" } + - contains: { $composite_id: $first_node_id } + - contains: { $composite_id: $first_index } diff --git a/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResults.java b/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResults.java index 739e2a8d91c40..f91e2beee1aae 100644 --- a/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResults.java +++ b/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResults.java @@ -88,7 +88,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); builder.field(ID_FIELD, key); - ShardProfileId shardProfileId = parseProfileShardId(key); + ShardProfileId shardProfileId = parseCompositeProfileShardId(key); if (shardProfileId != null) { builder.field(NODE_ID_FIELD, shardProfileId.nodeId()); builder.field(SHARD_ID_FIELD, shardProfileId.shardId()); @@ -222,10 +222,9 @@ record ShardProfileId(String nodeId, String indexName, int shardId, @Nullable St * @param compositeId see above for accepted formats * @return ShardProfileId with parsed components or null if the compositeId has an unsupported format */ - static ShardProfileId parseProfileShardId(String compositeId) { - if (Strings.isNullOrEmpty(compositeId)) { - return null; - } + static ShardProfileId parseCompositeProfileShardId(String compositeId) { + assert Strings.isNullOrEmpty(compositeId) == false : "An empty id should not be passed to parseCompositeProfileShardId"; + Pattern r = Pattern.compile("\\[([^]]+)\\]\\[([^]]+)\\]\\[(\\d+)\\]"); Matcher m = r.matcher(compositeId); if (m.find()) { @@ -241,6 +240,7 @@ static ShardProfileId parseProfileShardId(String compositeId) { } return new ShardProfileId(nodeId, indexName, shardId, cluster); } else { + assert false : "Unable to match input against expected pattern of [nodeId][indexName][shardId]. Input: " + compositeId; logger.warn("Unable to match input against expected pattern of [nodeId][indexName][shardId]. Input: {}", compositeId); return null; } diff --git a/server/src/test/java/org/elasticsearch/search/profile/SearchProfileResultsTests.java b/server/src/test/java/org/elasticsearch/search/profile/SearchProfileResultsTests.java index 0fbc0fa01dc04..92a5876d7228e 100644 --- a/server/src/test/java/org/elasticsearch/search/profile/SearchProfileResultsTests.java +++ b/server/src/test/java/org/elasticsearch/search/profile/SearchProfileResultsTests.java @@ -9,12 +9,17 @@ package org.elasticsearch.search.profile; import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.Maps; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.test.AbstractXContentSerializingTestCase; import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; import java.util.Map; +import java.util.UUID; import java.util.function.Predicate; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; @@ -32,29 +37,55 @@ public static SearchProfileResults createTestItem() { return new SearchProfileResults(shards); } - public void testParseProfileShardId() { - record TestPattern(String id, SearchProfileResults.ShardProfileId expected) {} + public void testParseCompositeProfileShardId() { + String indexUuid = UUID.randomUUID().toString(); // not part of composite ID, so can be anything - TestPattern[] cases = new TestPattern[] { - new TestPattern(null, null), - new TestPattern("", null), - new TestPattern("[][][]", null), - new TestPattern("chsk8ad", null), - new TestPattern("[UngEVXTBQL-7w5j_tftGAQ][remote1:blogs]", null), // shardId is missing - new TestPattern("[UngEVXTBQL-7w5j_tftGAQ][remote1:blogs][xyz]", null), // shardId must be integer + String nodeId1 = NodeEnvironment.generateNodeId(Settings.EMPTY); + String nodeId2 = NodeEnvironment.generateNodeId( + Settings.builder() + .put(randomAlphaOfLengthBetween(5, 18), randomNonNegativeInt()) + .put(randomAlphaOfLengthBetween(5, 18), randomAlphaOfLength(15)) + .build() + ); + int shardId1 = randomNonNegativeInt(); + int shardId2 = randomNonNegativeInt(); + int shardId3 = randomNonNegativeInt(); + + String indexName1 = "x"; + String indexName2 = ".ds-filebeat-8.3.2-2023.05.04-000420"; + String indexName3 = randomIdentifier(); + + record TestPattern(SearchShardTarget input, SearchProfileResults.ShardProfileId expected) {} + + TestPattern[] validPatterns = new TestPattern[] { new TestPattern( - "[UngEVXTBQL-7w5j_tftGAQ][remote1:blogs][0]", - new SearchProfileResults.ShardProfileId("UngEVXTBQL-7w5j_tftGAQ", "blogs", 0, "remote1") + new SearchShardTarget(nodeId1, new ShardId(indexName1, indexUuid, shardId1), "remote1"), + new SearchProfileResults.ShardProfileId(nodeId1, indexName1, shardId1, "remote1") ), new TestPattern( - "[2m7SW9oIRrirdrwirM1mwQ][.ds-filebeat-8.3.2-2023.05.04-000420][303]", - new SearchProfileResults.ShardProfileId("2m7SW9oIRrirdrwirM1mwQ", ".ds-filebeat-8.3.2-2023.05.04-000420", 303, null) + new SearchShardTarget(nodeId2, new ShardId(indexName2, indexUuid, shardId2), null), + new SearchProfileResults.ShardProfileId(nodeId2, indexName2, shardId2, null) ), - new TestPattern("[_na_][:indexName_1][303]", new SearchProfileResults.ShardProfileId("_na_", "indexName_1", 303, "")) }; + new TestPattern( + new SearchShardTarget(null, new ShardId(indexName3, indexUuid, shardId3), null), + new SearchProfileResults.ShardProfileId("_na_", indexName3, shardId3, null) + ) }; + for (TestPattern testPattern : validPatterns) { + assertEquals(testPattern.expected, SearchProfileResults.parseCompositeProfileShardId(testPattern.input.toString())); + } + } - for (TestPattern testPattern : cases) { - assertEquals(testPattern.expected, SearchProfileResults.parseProfileShardId(testPattern.id)); + public void testParseCompositeProfileShardIdWithInvalidEntries() { + String[] invalidPatterns = new String[] { + null, + "", + "chsk8ad", + "[UngEVXTBQL-7w5j_tftGAQ][remote1:blogs]", // shardId is missing + "[UngEVXTBQL-7w5j_tftGAQ][remote1:blogs][xyz]" // shardId must be integer + }; + for (String testPattern : invalidPatterns) { + expectThrows(AssertionError.class, () -> SearchProfileResults.parseCompositeProfileShardId(testPattern)); } } From 6548ac108e591e68df7c3adbd841f4fff1a38406 Mon Sep 17 00:00:00 2001 From: Michael Peterson Date: Fri, 16 Jun 2023 16:35:06 -0400 Subject: [PATCH 6/8] SearchProfileResultsTests.createTestItem now properly uses SearchShardTarget.toString to set the key for the SearchProfileShardResult map --- .../search/profile/SearchProfileResultsTests.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/search/profile/SearchProfileResultsTests.java b/server/src/test/java/org/elasticsearch/search/profile/SearchProfileResultsTests.java index 92a5876d7228e..f02114a48991c 100644 --- a/server/src/test/java/org/elasticsearch/search/profile/SearchProfileResultsTests.java +++ b/server/src/test/java/org/elasticsearch/search/profile/SearchProfileResultsTests.java @@ -32,7 +32,12 @@ public static SearchProfileResults createTestItem() { for (int i = 0; i < size; i++) { SearchProfileQueryPhaseResult searchResult = SearchProfileQueryPhaseResultTests.createTestItem(); ProfileResult fetchResult = randomBoolean() ? null : ProfileResultTests.createTestItem(2); - shards.put(randomAlphaOfLengthBetween(5, 10), new SearchProfileShardResult(searchResult, fetchResult)); + SearchShardTarget target = new SearchShardTarget( + NodeEnvironment.generateNodeId(Settings.EMPTY), + new ShardId(randomIdentifier(), UUID.randomUUID().toString(), randomNonNegativeInt()), + null + ); + shards.put(target.toString(), new SearchProfileShardResult(searchResult, fetchResult)); } return new SearchProfileResults(shards); } From 0489321ff3825c7522534558b2ab4c89a339505e Mon Sep 17 00:00:00 2001 From: Michael Peterson Date: Fri, 23 Jun 2023 08:57:06 -0400 Subject: [PATCH 7/8] PR feedback - updated yaml test skip to 8.9.99 and changed compositeShardId regex to private static final var --- .../resources/rest-api-spec/test/search/370_profile.yml | 4 ++-- .../elasticsearch/search/profile/SearchProfileResults.java | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml index ea40cebcc25eb..4fd6dadbdbb36 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml @@ -404,8 +404,8 @@ dfs profile for search with dfs_query_then_fetch: --- composite_id_parsed: - skip: - version: " - 8.8.99" - reason: node_id, shard_id, index and cluster fields added in 8.9.0 + version: " - 8.9.99" + reason: node_id, shard_id, index and cluster fields added in 8.10.0 features: "contains" - do: diff --git a/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResults.java b/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResults.java index f91e2beee1aae..8c19fb0ff9f99 100644 --- a/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResults.java +++ b/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResults.java @@ -207,6 +207,8 @@ private static void parseProfileResultsEntry(XContentParser parser, Map Date: Fri, 23 Jun 2023 19:58:37 -0400 Subject: [PATCH 8/8] when running when with remote cluster the cluster changes, so can't match on known value --- .../resources/rest-api-spec/test/search/370_profile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml index 4fd6dadbdbb36..0adc7e0caf684 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml @@ -425,6 +425,6 @@ composite_id_parsed: - length: { profile.shards.0.node_id: 22 } - gte: { profile.shards.0.shard_id: 0 } - match: { profile.shards.0.index: "test" } - - match: { profile.shards.0.cluster: "(local)" } + - is_true: profile.shards.0.cluster - contains: { $composite_id: $first_node_id } - contains: { $composite_id: $first_index }