From 80288389912bb799f2f0eba10b39b0b136bfa5d6 Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Tue, 6 Apr 2021 15:03:40 -0700 Subject: [PATCH 1/6] Support fetching _tier values. --- .../mapper/DataTierFieldMapper.java | 28 ++++++++++++++----- .../mapper/DataTierFieldTypeTests.java | 14 ++++++++++ 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java index e56ebc55f5ee0..7bda4160f4956 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java @@ -12,6 +12,7 @@ import org.apache.lucene.search.Query; import org.elasticsearch.common.Strings; import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.ConstantFieldType; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MetadataFieldMapper; @@ -21,6 +22,8 @@ import java.util.Collections; +import static java.util.Collections.singletonList; + public class DataTierFieldMapper extends MetadataFieldMapper { public static final String NAME = "_tier"; @@ -52,19 +55,16 @@ protected boolean matches(String pattern, boolean caseInsensitive, SearchExecuti if (caseInsensitive) { pattern = Strings.toLowercaseAscii(pattern); } - String tierPreference = DataTierAllocationDecider.INDEX_ROUTING_PREFER_SETTING.get(context.getIndexSettings().getSettings()); + String tierPreference = getTierPreference(context); if (tierPreference == null) { return false; } - // Tier preference can be a comma-delimited list of tiers, ordered by preference - // It was decided we should only test the first of these potentially multiple preferences. - String firstPreference = tierPreference.split(",")[0].trim(); - return Regex.simpleMatch(pattern, firstPreference); + return Regex.simpleMatch(pattern, tierPreference); } @Override public Query existsQuery(SearchExecutionContext context) { - String tierPreference = DataTierAllocationDecider.INDEX_ROUTING_PREFER_SETTING.get(context.getIndexSettings().getSettings()); + String tierPreference = getTierPreference(context); if (tierPreference == null) { return new MatchNoDocsQuery(); } @@ -73,7 +73,21 @@ public Query existsQuery(SearchExecutionContext context) { @Override public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); + String tierPreference = getTierPreference(context); + return lookup -> singletonList(tierPreference); + } + + private String getTierPreference(SearchExecutionContext context) { + Settings settings = context.getIndexSettings().getSettings(); + String value = DataTierAllocationDecider.INDEX_ROUTING_PREFER_SETTING.get(context.getIndexSettings().getSettings()); + + if (value == null) { + return null; + } + + // Tier preference can be a comma-delimited list of tiers, ordered by preference + // It was decided we should only test the first of these potentially multiple preferences. + return value.split(",")[0].trim(); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldTypeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldTypeTests.java index 9f318bec9b4e7..f4d211805e7a1 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldTypeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldTypeTests.java @@ -14,10 +14,15 @@ import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.mapper.DocCountFieldMapper; +import org.elasticsearch.index.mapper.FieldTypeTestCase; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperServiceTestCase; +import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider; import java.io.IOException; @@ -66,6 +71,15 @@ public void testRegexpQuery() { assertThat(e.getMessage(), containsString("Can only use regexp queries on keyword and text fields")); } + public void testFetchValue() throws IOException { + MappedFieldType ft = DataTierFieldMapper.DataTierFieldType.INSTANCE; + SearchExecutionContext context = createContext(); + SourceLookup lookup = new SourceLookup(); + + ValueFetcher valueFetcher = ft.valueFetcher(context, null); + assertEquals(Arrays.asList("data_warm"), valueFetcher.fetchValues(lookup)); + } + private SearchExecutionContext createContext() { IndexMetadata indexMetadata = IndexMetadata.builder("index") .settings( From b0f38a548b5dfad14a58f10074ebf400496168df Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Tue, 6 Apr 2021 15:05:21 -0700 Subject: [PATCH 2/6] Reorganize metadata fields docs. --- docs/reference/mapping/fields.asciidoc | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/reference/mapping/fields.asciidoc b/docs/reference/mapping/fields.asciidoc index 80778ffa906f8..6a2b4d35b9fd6 100644 --- a/docs/reference/mapping/fields.asciidoc +++ b/docs/reference/mapping/fields.asciidoc @@ -13,11 +13,6 @@ fields can be customized when a mapping is created. The index to which the document belongs. -<>:: - - The current data tier preference of the index to which the document belongs. - - <>:: The document's ID. @@ -67,6 +62,10 @@ fields can be customized when a mapping is created. Application specific metadata. +<>:: + + The current data tier preference of the index to which the document belongs. + include::fields/doc-count-field.asciidoc[] include::fields/field-names-field.asciidoc[] @@ -77,10 +76,10 @@ include::fields/id-field.asciidoc[] include::fields/index-field.asciidoc[] -include::fields/tier-field.asciidoc[] - include::fields/meta-field.asciidoc[] include::fields/routing-field.asciidoc[] include::fields/source-field.asciidoc[] + +include::fields/tier-field.asciidoc[] From f7f49416639603ada92563973a60c95997581b2f Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Tue, 6 Apr 2021 15:44:13 -0700 Subject: [PATCH 3/6] Fix checkstyle. --- .../routing/allocation/mapper/DataTierFieldTypeTests.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldTypeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldTypeTests.java index f4d211805e7a1..b77dbc6c89fb7 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldTypeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldTypeTests.java @@ -14,10 +14,7 @@ import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.mapper.DocCountFieldMapper; -import org.elasticsearch.index.mapper.FieldTypeTestCase; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperServiceTestCase; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardException; From cad81c79870e3e3835c3abd3c6813482ee4b64b2 Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Tue, 6 Apr 2021 23:01:56 -0700 Subject: [PATCH 4/6] Fix bad refactor in getTierPreference. --- .../cluster/routing/allocation/mapper/DataTierFieldMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java index 7bda4160f4956..22c49f80aa188 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java @@ -79,7 +79,7 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) private String getTierPreference(SearchExecutionContext context) { Settings settings = context.getIndexSettings().getSettings(); - String value = DataTierAllocationDecider.INDEX_ROUTING_PREFER_SETTING.get(context.getIndexSettings().getSettings()); + String value = DataTierAllocationDecider.INDEX_ROUTING_PREFER_SETTING.get(settings); if (value == null) { return null; From 15fe293f1e50a432573c21222dad180e62fac804 Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Tue, 6 Apr 2021 23:03:45 -0700 Subject: [PATCH 5/6] Reject non-null formats. --- .../routing/allocation/mapper/DataTierFieldMapper.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java index 22c49f80aa188..c21754dc29aa6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java @@ -73,6 +73,10 @@ public Query existsQuery(SearchExecutionContext context) { @Override public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + String tierPreference = getTierPreference(context); return lookup -> singletonList(tierPreference); } From 3da92149b5c4e689a5914f9f05f31ce31b4608a7 Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Thu, 8 Apr 2021 10:22:49 -0700 Subject: [PATCH 6/6] Make sure to handle missing setting. --- .../allocation/mapper/DataTierFieldMapper.java | 17 +++++++++++------ .../mapper/DataTierFieldTypeTests.java | 9 ++++++--- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java index 79d655bc86180..45da755e63fd2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java @@ -21,8 +21,7 @@ import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider; import java.util.Collections; - -import static java.util.Collections.singletonList; +import java.util.List; public class DataTierFieldMapper extends MetadataFieldMapper { @@ -57,7 +56,7 @@ protected boolean matches(String pattern, boolean caseInsensitive, SearchExecuti } String tierPreference = getTierPreference(context); - if (Strings.hasText(tierPreference) == false) { + if (tierPreference == null) { return false; } return Regex.simpleMatch(pattern, tierPreference); @@ -66,7 +65,7 @@ protected boolean matches(String pattern, boolean caseInsensitive, SearchExecuti @Override public Query existsQuery(SearchExecutionContext context) { String tierPreference = getTierPreference(context); - if (Strings.hasText(tierPreference) == false) { + if (tierPreference == null) { return new MatchNoDocsQuery(); } return new MatchAllDocsQuery(); @@ -79,14 +78,20 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) } String tierPreference = getTierPreference(context); - return lookup -> singletonList(tierPreference); + return tierPreference == null + ? lookup -> List.of() + : lookup -> List.of(tierPreference); } + /** + * Retrieve the first tier preference from the index setting. If the setting is not + * present, then return null. + */ private String getTierPreference(SearchExecutionContext context) { Settings settings = context.getIndexSettings().getSettings(); String value = DataTierAllocationDecider.INDEX_ROUTING_PREFER_SETTING.get(settings); - if (value == null) { + if (Strings.hasText(value) == false) { return null; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldTypeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldTypeTests.java index 5802f8f6a6e0d..586e69223dc97 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldTypeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldTypeTests.java @@ -27,6 +27,7 @@ import java.util.function.Predicate; import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonList; import static org.hamcrest.Matchers.containsString; public class DataTierFieldTypeTests extends MapperServiceTestCase { @@ -85,11 +86,13 @@ public void testRegexpQuery() { public void testFetchValue() throws IOException { MappedFieldType ft = DataTierFieldMapper.DataTierFieldType.INSTANCE; - SearchExecutionContext context = createContext(); SourceLookup lookup = new SourceLookup(); - ValueFetcher valueFetcher = ft.valueFetcher(context, null); - assertEquals(Arrays.asList("data_warm"), valueFetcher.fetchValues(lookup)); + ValueFetcher valueFetcher = ft.valueFetcher(createContext(), null); + assertEquals(singletonList("data_warm"), valueFetcher.fetchValues(lookup)); + + ValueFetcher emptyValueFetcher = ft.valueFetcher(createContextWithoutSetting(), null); + assertTrue(emptyValueFetcher.fetchValues(lookup).isEmpty()); } private SearchExecutionContext createContext() {