diff --git a/docs/changelog/103648.yaml b/docs/changelog/103648.yaml new file mode 100644 index 0000000000000..d4fa489a6812c --- /dev/null +++ b/docs/changelog/103648.yaml @@ -0,0 +1,5 @@ +pr: 103648 +summary: Introduce experimental pass-through field type +area: TSDB +type: enhancement +issues: [] diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java index 694e015b602f8..88e529ec5569b 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java @@ -26,6 +26,7 @@ import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MappingParserContext; +import org.elasticsearch.index.mapper.PassThroughObjectMapper; import java.io.IOException; import java.io.UncheckedIOException; @@ -152,8 +153,9 @@ private List findRoutingPaths(String indexName, Settings allSettings, Li .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, dummyShards) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, shardReplicas) .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) + .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) // Avoid failing because index.routing_path is missing - .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) + .putList(INDEX_ROUTING_PATH.getKey(), List.of("path")) .build(); tmpIndexMetadata.settings(finalResolvedSettings); @@ -164,6 +166,13 @@ private List findRoutingPaths(String indexName, Settings allSettings, Li for (var fieldMapper : mapperService.documentMapper().mappers().fieldMappers()) { extractPath(routingPaths, fieldMapper); } + for (var objectMapper : mapperService.documentMapper().mappers().objectMappers().values()) { + if (objectMapper instanceof PassThroughObjectMapper passThroughObjectMapper) { + if (passThroughObjectMapper.containsDimensions()) { + routingPaths.add(passThroughObjectMapper.fullPath() + ".*"); + } + } + } for (var template : mapperService.getAllDynamicTemplates()) { if (template.pathMatch().isEmpty()) { continue; diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java index db0e3e5cd6258..c65854903f7a9 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java @@ -638,6 +638,33 @@ public void testGenerateRoutingPathFromDynamicTemplate_nonKeywordTemplate() thro assertEquals(2, IndexMetadata.INDEX_ROUTING_PATH.get(result).size()); } + public void testGenerateRoutingPathFromPassThroughObject() throws Exception { + Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); + String mapping = """ + { + "_doc": { + "properties": { + "labels": { + "type": "passthrough", + "time_series_dimension": true + }, + "metrics": { + "type": "passthrough" + }, + "another_field": { + "type": "keyword" + } + } + } + } + """; + Settings result = generateTsdbSettings(mapping, now); + assertThat(result.size(), equalTo(3)); + assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); + assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); + assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("labels.*")); + } + private Settings generateTsdbSettings(String mapping, Instant now) throws IOException { Metadata metadata = Metadata.EMPTY_METADATA; String dataStreamName = "logs-app1"; diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index 6a84f08a2c193..78cbc3e72101f 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -191,3 +191,262 @@ index without timestamp with pipeline: pipeline: my_pipeline body: - '{"@timestamp": "wrong_format", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' + +--- +dynamic templates: + - skip: + version: " - 8.12.99" + reason: "Support for dynamic fields was added in 8.13" + - do: + indices.put_index_template: + name: my-dynamic-template + body: + index_patterns: [k9s*] + data_stream: {} + template: + settings: + index: + number_of_shards: 1 + mode: time_series + time_series: + start_time: 2023-08-31T13:03:08.138Z + + mappings: + properties: + attributes: + type: passthrough + dynamic: true + time_series_dimension: true + dynamic_templates: + - counter_metric: + mapping: + type: integer + time_series_metric: counter + + - do: + bulk: + index: k9s + refresh: true + body: + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "attributes.dim": "A", "attributes.another.dim": "C" }' + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:09.138Z","data": "20", "attributes.dim": "A", "attributes.another.dim": "C" }' + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:10.138Z","data": "30", "attributes.dim": "B", "attributes.another.dim": "D" }' + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:10.238Z","data": "40", "attributes.dim": "B", "attributes.another.dim": "D" }' + + - do: + search: + index: k9s + body: + size: 0 + + - match: { hits.total.value: 4 } + + - do: + search: + index: k9s + body: + size: 0 + aggs: + filterA: + filter: + term: + dim: A + aggs: + tsids: + terms: + field: _tsid + + - length: { aggregations.filterA.tsids.buckets: 1 } + - match: { aggregations.filterA.tsids.buckets.0.key: { "attributes.another.dim": "C", "attributes.dim": "A" } } + - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } + + - do: + search: + index: k9s + body: + size: 0 + aggs: + filterA: + filter: + term: + another.dim: C + aggs: + tsids: + terms: + field: _tsid + + - length: { aggregations.filterA.tsids.buckets: 1 } + - match: { aggregations.filterA.tsids.buckets.0.key: { "attributes.another.dim": "C", "attributes.dim": "A" } } + - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } + +--- +dynamic templates - conflicting aliases: + - skip: + version: " - 8.12.99" + reason: "Support for dynamic fields was added in 8.13" + - do: + indices.put_index_template: + name: my-dynamic-template + body: + index_patterns: [k9s*] + data_stream: {} + template: + settings: + index: + number_of_shards: 1 + mode: time_series + time_series: + start_time: 2023-08-31T13:03:08.138Z + + mappings: + properties: + attributes: + type: passthrough + dynamic: true + time_series_dimension: true + resource_attributes: + type: passthrough + dynamic: true + time_series_dimension: true + dynamic_templates: + - counter_metric: + mapping: + type: integer + time_series_metric: counter + + - do: + bulk: + index: k9s + refresh: true + body: + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "attributes.dim": "A", "resource_attributes.dim": "C" }' + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:09.138Z","data": "20", "attributes.dim": "A", "resource_attributes.dim": "C" }' + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:10.138Z","data": "30", "attributes.dim": "B", "resource_attributes.dim": "D" }' + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:10.238Z","data": "40", "attributes.dim": "B", "resource_attributes.dim": "D" }' + + - do: + search: + index: k9s + body: + size: 0 + + - match: { hits.total.value: 4 } + + - do: + search: + index: k9s + body: + size: 0 + aggs: + filterA: + filter: + term: + dim: "C" + aggs: + tsids: + terms: + field: _tsid + + - length: { aggregations.filterA.tsids.buckets: 1 } + - match: { aggregations.filterA.tsids.buckets.0.key: { "resource_attributes.dim": "C", "attributes.dim": "A" } } + - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } + + - do: + search: + index: k9s + body: + size: 0 + aggs: + filterA: + filter: + term: + attributes.dim: A + aggs: + tsids: + terms: + field: _tsid + + - length: { aggregations.filterA.tsids.buckets: 1 } + - match: { aggregations.filterA.tsids.buckets.0.key: { "resource_attributes.dim": "C", "attributes.dim": "A" } } + - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } + +--- +dynamic templates - subobject in passthrough object error: + - skip: + version: " - 8.12.99" + reason: "Support for dynamic fields was added in 8.13" + - do: + catch: /Tried to add subobject \[subcategory\] to object \[attributes\] which does not support subobjects/ + indices.put_index_template: + name: my-dynamic-template + body: + index_patterns: [k9s*] + data_stream: {} + template: + settings: + index: + mode: time_series + + mappings: + properties: + attributes: + type: passthrough + properties: + subcategory: + type: object + properties: + dim: + type: keyword + + - do: + catch: /Mapping definition for \[attributes\] has unsupported parameters:\ \[subobjects \:\ true\]/ + indices.put_index_template: + name: my-dynamic-template + body: + index_patterns: [k9s*] + data_stream: {} + template: + settings: + index: + number_of_shards: 1 + mode: time_series + time_series: + start_time: 2023-08-31T13:03:08.138Z + + mappings: + properties: + attributes: + type: passthrough + subobjects: true + +--- +dynamic templates - passthrough not under root error: + - skip: + version: " - 8.12.99" + reason: "Support for dynamic fields was added in 8.13" + - do: + catch: /Tried to add passthrough subobject \[attributes\] to object \[resource\], passthrough is not supported as a subobject/ + indices.put_index_template: + name: my-dynamic-template + body: + index_patterns: [k9s*] + data_stream: {} + template: + settings: + index: + mode: time_series + + mappings: + properties: + "resource.attributes": + type: passthrough + dynamic: true + time_series_dimension: true diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index 0d40bd2d08c14..1ed9d759c4ca8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -248,6 +248,10 @@ public static class ExtractFromSource extends IndexRouting { this.parserConfig = XContentParserConfiguration.EMPTY.withFiltering(Set.copyOf(routingPaths), null, true); } + public boolean matchesField(String fieldName) { + return isRoutingPath.test(fieldName); + } + @Override public void process(IndexRequest indexRequest) {} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index b9dfc83d17683..c0d15da49225e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -552,7 +552,12 @@ public final MapperBuilderContext createDynamicMapperBuilderContext() { if (p.endsWith(".")) { p = p.substring(0, p.length() - 1); } - return new MapperBuilderContext(p, mappingLookup().isSourceSynthetic(), false); + boolean containsDimensions = false; + ObjectMapper objectMapper = mappingLookup.objectMappers().get(p); + if (objectMapper instanceof PassThroughObjectMapper passThroughObjectMapper) { + containsDimensions = passThroughObjectMapper.containsDimensions(); + } + return new MapperBuilderContext(p, mappingLookup().isSourceSynthetic(), false, containsDimensions); } public abstract XContentParser parser(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java index f2d1b8058f115..813fb1a051000 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java @@ -307,20 +307,34 @@ private static final class Concrete implements Strategy { this.parseField = parseField; } - void createDynamicField(Mapper.Builder builder, DocumentParserContext context) throws IOException { - Mapper mapper = builder.build(context.createDynamicMapperBuilderContext()); + void createDynamicField(Mapper.Builder builder, DocumentParserContext context, MapperBuilderContext mapperBuilderContext) + throws IOException { + Mapper mapper = builder.build(mapperBuilderContext); context.addDynamicMapper(mapper); parseField.accept(context, mapper); } + void createDynamicField(Mapper.Builder builder, DocumentParserContext context) throws IOException { + createDynamicField(builder, context, context.createDynamicMapperBuilderContext()); + } + @Override public void newDynamicStringField(DocumentParserContext context, String name) throws IOException { - createDynamicField( - new TextFieldMapper.Builder(name, context.indexAnalyzers()).addMultiField( - new KeywordFieldMapper.Builder("keyword", context.indexSettings().getIndexVersionCreated()).ignoreAbove(256) - ), - context - ); + MapperBuilderContext mapperBuilderContext = context.createDynamicMapperBuilderContext(); + if (mapperBuilderContext.parentObjectContainsDimensions()) { + createDynamicField( + new KeywordFieldMapper.Builder(name, context.indexSettings().getIndexVersionCreated()), + context, + mapperBuilderContext + ); + } else { + createDynamicField( + new TextFieldMapper.Builder(name, context.indexAnalyzers()).addMultiField( + new KeywordFieldMapper.Builder("keyword", context.indexSettings().getIndexVersionCreated()).ignoreAbove(256) + ), + context + ); + } } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 56a50c2dee0aa..8d529853f46bb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -166,6 +166,9 @@ protected Parameter[] getParameters() { @Override public IpFieldMapper build(MapperBuilderContext context) { + if (context.parentObjectContainsDimensions()) { + dimension.setValue(true); + } return new IpFieldMapper( name, new IpFieldType( diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index c3a740e4cbfe6..b0d47314159fb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -299,6 +299,9 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType } else if (splitQueriesOnWhitespace.getValue()) { searchAnalyzer = Lucene.WHITESPACE_ANALYZER; } + if (context.parentObjectContainsDimensions()) { + dimension(true); + } return new KeywordFieldType( context.buildFullName(name), fieldType, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java index 7506e8b8f6671..4154c936bab52 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java @@ -19,17 +19,23 @@ public class MapperBuilderContext { * The root context, to be used when building a tree of mappers */ public static MapperBuilderContext root(boolean isSourceSynthetic, boolean isDataStream) { - return new MapperBuilderContext(null, isSourceSynthetic, isDataStream); + return new MapperBuilderContext(null, isSourceSynthetic, isDataStream, false); } private final String path; private final boolean isSourceSynthetic; private final boolean isDataStream; + private final boolean parentObjectContainsDimensions; - MapperBuilderContext(String path, boolean isSourceSynthetic, boolean isDataStream) { + MapperBuilderContext(String path) { + this(path, false, false, false); + } + + MapperBuilderContext(String path, boolean isSourceSynthetic, boolean isDataStream, boolean parentObjectContainsDimensions) { this.path = path; this.isSourceSynthetic = isSourceSynthetic; this.isDataStream = isDataStream; + this.parentObjectContainsDimensions = parentObjectContainsDimensions; } /** @@ -38,7 +44,7 @@ public static MapperBuilderContext root(boolean isSourceSynthetic, boolean isDat * @return a new MapperBuilderContext with this context as its parent */ public MapperBuilderContext createChildContext(String name) { - return new MapperBuilderContext(buildFullName(name), isSourceSynthetic, isDataStream); + return new MapperBuilderContext(buildFullName(name), isSourceSynthetic, isDataStream, parentObjectContainsDimensions); } /** @@ -64,4 +70,12 @@ public boolean isSourceSynthetic() { public boolean isDataStream() { return isDataStream; } + + /** + * Are these field mappings being built dimensions? + */ + public boolean parentObjectContainsDimensions() { + return parentObjectContainsDimensions; + } + } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java index 268d028be91a6..a654819811621 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java @@ -118,7 +118,7 @@ private static class NestedMapperBuilderContext extends MapperBuilderContext { final boolean parentIncludedInRoot; NestedMapperBuilderContext(String path, boolean parentIncludedInRoot) { - super(path, false, false); + super(path); this.parentIncludedInRoot = parentIncludedInRoot; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index d25832a28d318..36ce9e51f74a0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -267,6 +267,10 @@ protected Parameter[] getParameters() { @Override public NumberFieldMapper build(MapperBuilderContext context) { + if (context.parentObjectContainsDimensions()) { + dimension.setValue(true); + } + MappedFieldType ft = new NumberFieldType(context.buildFullName(name), this); return new NumberFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo, context.isSourceSynthetic(), this); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index 6ced2b49bb84a..1ed9713d73e75 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -313,12 +313,21 @@ protected static void parseProperties( + "] which does not support subobjects" ); } + if (type.equals(PassThroughObjectMapper.CONTENT_TYPE) && objBuilder instanceof RootObjectMapper.Builder == false) { + throw new MapperParsingException( + "Tried to add passthrough subobject [" + + fieldName + + "] to object [" + + objBuilder.name() + + "], passthrough is not supported as a subobject" + ); + } Mapper.TypeParser typeParser = parserContext.typeParser(type); if (typeParser == null) { throw new MapperParsingException("No handler for type [" + type + "] declared on field [" + fieldName + "]"); } Mapper.Builder fieldBuilder; - if (objBuilder.subobjects.value() == false) { + if (objBuilder.subobjects.value() == false || type.equals(FieldAliasMapper.CONTENT_TYPE)) { fieldBuilder = typeParser.parse(fieldName, propNode, parserContext); } else { String[] fieldNameParts = fieldName.split("\\."); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java new file mode 100644 index 0000000000000..7688b217ab7fc --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java @@ -0,0 +1,155 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.common.Explicit; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.mapper.MapperService.MergeReason; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Locale; +import java.util.Map; + +import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue; + +/** + * Mapper for pass-through objects. + * + * Pass-through objects allow creating fields inside them that can also be referenced directly in search queries. + * They also include parameters that affect how nested fields get configured. For instance, if parameter "time_series_dimension" + * is set, eligible subfields are marked as dimensions and keyword fields are additionally included in routing and tsid calculations. + */ +public class PassThroughObjectMapper extends ObjectMapper { + public static final String CONTENT_TYPE = "passthrough"; + + public static class Builder extends ObjectMapper.Builder { + + // Controls whether subfields are configured as time-series dimensions. + protected Explicit timeSeriesDimensionSubFields = Explicit.IMPLICIT_FALSE; + + public Builder(String name) { + // Subobjects are not currently supported. + super(name, Explicit.IMPLICIT_FALSE); + } + + @Override + public PassThroughObjectMapper.Builder add(Mapper.Builder builder) { + if (timeSeriesDimensionSubFields.value() && builder instanceof KeywordFieldMapper.Builder keywordBuilder) { + keywordBuilder.dimension(true); + } + super.add(builder); + return this; + } + + public PassThroughObjectMapper.Builder setContainsDimensions() { + timeSeriesDimensionSubFields = Explicit.EXPLICIT_TRUE; + return this; + } + + @Override + public PassThroughObjectMapper build(MapperBuilderContext context) { + return new PassThroughObjectMapper( + name, + enabled, + dynamic, + buildMappers(context.createChildContext(name)), + timeSeriesDimensionSubFields + ); + } + } + + // If set, its subfields are marked as time-series dimensions (for the types supporting this). + private final Explicit timeSeriesDimensionSubFields; + + PassThroughObjectMapper( + String name, + Explicit enabled, + Dynamic dynamic, + Map mappers, + Explicit timeSeriesDimensionSubFields + ) { + // Subobjects are not currently supported. + super(name, name, enabled, Explicit.IMPLICIT_FALSE, dynamic, mappers); + this.timeSeriesDimensionSubFields = timeSeriesDimensionSubFields; + } + + @Override + PassThroughObjectMapper withoutMappers() { + return new PassThroughObjectMapper(simpleName(), enabled, dynamic, Map.of(), timeSeriesDimensionSubFields); + } + + public boolean containsDimensions() { + return timeSeriesDimensionSubFields.value(); + } + + @Override + public PassThroughObjectMapper.Builder newBuilder(IndexVersion indexVersionCreated) { + PassThroughObjectMapper.Builder builder = new PassThroughObjectMapper.Builder(name()); + builder.enabled = enabled; + builder.dynamic = dynamic; + builder.timeSeriesDimensionSubFields = timeSeriesDimensionSubFields; + return builder; + } + + public PassThroughObjectMapper merge(ObjectMapper mergeWith, MergeReason reason, MapperMergeContext parentBuilderContext) { + final var mergeResult = MergeResult.build(this, mergeWith, reason, parentBuilderContext); + PassThroughObjectMapper mergeWithObject = (PassThroughObjectMapper) mergeWith; + + final Explicit containsDimensions = (mergeWithObject.timeSeriesDimensionSubFields.explicit()) + ? mergeWithObject.timeSeriesDimensionSubFields + : this.timeSeriesDimensionSubFields; + + return new PassThroughObjectMapper( + simpleName(), + mergeResult.enabled(), + mergeResult.dynamic(), + mergeResult.mappers(), + containsDimensions + ); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(simpleName()); + builder.field("type", CONTENT_TYPE); + if (timeSeriesDimensionSubFields.explicit()) { + builder.field(TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM, timeSeriesDimensionSubFields.value()); + } + if (dynamic != null) { + builder.field("dynamic", dynamic.name().toLowerCase(Locale.ROOT)); + } + if (isEnabled() != Defaults.ENABLED) { + builder.field("enabled", enabled.value()); + } + serializeMappers(builder, params); + return builder.endObject(); + } + + public static class TypeParser extends ObjectMapper.TypeParser { + @Override + public Mapper.Builder parse(String name, Map node, MappingParserContext parserContext) + throws MapperParsingException { + PassThroughObjectMapper.Builder builder = new Builder(name); + parsePassthrough(name, node, builder); + parseObjectFields(node, parserContext, builder); + return builder; + } + + protected static void parsePassthrough(String name, Map node, PassThroughObjectMapper.Builder builder) { + Object fieldNode = node.get(TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM); + if (fieldNode != null) { + builder.timeSeriesDimensionSubFields = Explicit.explicitBoolean( + nodeBooleanValue(fieldNode, name + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM) + ); + node.remove(TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM); + } + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java index 5d719ae4f5da7..8c4fae79c797c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java @@ -18,6 +18,8 @@ import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.DynamicTemplate.XContentFieldType; import org.elasticsearch.index.mapper.MapperService.MergeReason; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; @@ -74,6 +76,8 @@ public static class Builder extends ObjectMapper.Builder { protected Explicit dateDetection = Defaults.DATE_DETECTION; protected Explicit numericDetection = Defaults.NUMERIC_DETECTION; + private static final Logger logger = LogManager.getLogger(RootObjectMapper.Builder.class); + public Builder(String name, Explicit subobjects) { super(name, subobjects); } @@ -106,12 +110,14 @@ public RootObjectMapper.Builder addRuntimeFields(Map runti @Override public RootObjectMapper build(MapperBuilderContext context) { + Map mappers = buildMappers(context); + mappers.putAll(getAliasMappers(mappers, context)); return new RootObjectMapper( name, enabled, subobjects, dynamic, - buildMappers(context), + mappers, new HashMap<>(runtimeFields), dynamicDateTimeFormatters, dynamicTemplates, @@ -119,6 +125,39 @@ public RootObjectMapper build(MapperBuilderContext context) { numericDetection ); } + + Map getAliasMappers(Map mappers, MapperBuilderContext context) { + Map aliasMappers = new HashMap<>(); + for (Mapper mapper : mappers.values()) { + // Create aliases for all fields in child passthrough mappers and place them under the root object. + if (mapper instanceof PassThroughObjectMapper passthroughMapper) { + for (Mapper internalMapper : passthroughMapper.mappers.values()) { + if (internalMapper instanceof FieldMapper fieldMapper) { + // If there's a conflicting alias with the same name at the root level, we don't want to throw an error + // to avoid indexing disruption. + // TODO: record an error without affecting document indexing, so that it can be investigated later. + Mapper conflict = mappers.get(fieldMapper.simpleName()); + if (conflict != null) { + if (conflict.typeName().equals(FieldAliasMapper.CONTENT_TYPE) == false + || ((FieldAliasMapper) conflict).path().equals(fieldMapper.mappedFieldType.name()) == false) { + logger.warn( + "Root alias for field " + + fieldMapper.name() + + " conflicts with existing field or alias, skipping alias creation." + ); + } + } else { + FieldAliasMapper aliasMapper = new FieldAliasMapper.Builder(fieldMapper.simpleName()).path( + fieldMapper.mappedFieldType.name() + ).build(context); + aliasMappers.put(aliasMapper.simpleName(), aliasMapper); + } + } + } + } + } + return aliasMappers; + } } private final Explicit dynamicDateTimeFormatters; diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index 181852c2c3bc9..795ed2120b098 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -52,6 +52,7 @@ import org.elasticsearch.index.mapper.NestedPathFieldMapper; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.ObjectMapper; +import org.elasticsearch.index.mapper.PassThroughObjectMapper; import org.elasticsearch.index.mapper.RangeType; import org.elasticsearch.index.mapper.RoutingFieldMapper; import org.elasticsearch.index.mapper.RuntimeField; @@ -193,6 +194,7 @@ public static Map getMappers(List mappe mappers.put(KeywordFieldMapper.CONTENT_TYPE, KeywordFieldMapper.PARSER); mappers.put(ObjectMapper.CONTENT_TYPE, new ObjectMapper.TypeParser()); mappers.put(NestedObjectMapper.CONTENT_TYPE, new NestedObjectMapper.TypeParser()); + mappers.put(PassThroughObjectMapper.CONTENT_TYPE, new PassThroughObjectMapper.TypeParser()); mappers.put(TextFieldMapper.CONTENT_TYPE, TextFieldMapper.PARSER); mappers.put(DenseVectorFieldMapper.CONTENT_TYPE, DenseVectorFieldMapper.PARSER); diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 773934615e051..54e6b597c826c 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -33,7 +33,9 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; import org.elasticsearch.index.mapper.IdLoader; +import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.NestedLookup; import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.query.AbstractQueryBuilder; @@ -74,6 +76,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.Executor; import java.util.concurrent.ThreadPoolExecutor; import java.util.function.LongSupplier; @@ -891,7 +895,22 @@ public SourceLoader newSourceLoader() { public IdLoader newIdLoader() { if (indexService.getIndexSettings().getMode() == IndexMode.TIME_SERIES) { var indexRouting = (IndexRouting.ExtractFromSource) indexService.getIndexSettings().getIndexRouting(); - return IdLoader.createTsIdLoader(indexRouting, indexService.getMetadata().getRoutingPaths()); + List routingPaths = indexService.getMetadata().getRoutingPaths(); + for (String routingField : routingPaths) { + if (routingField.contains("*")) { + // In case the routing fields include path matches, find any matches and add them as distinct fields + // to the routing path. + Set matchingRoutingPaths = new TreeSet<>(routingPaths); + for (Mapper mapper : indexService.mapperService().mappingLookup().fieldMappers()) { + if (mapper instanceof KeywordFieldMapper && indexRouting.matchesField(mapper.name())) { + matchingRoutingPaths.add(mapper.name()); + } + } + routingPaths = new ArrayList<>(matchingRoutingPaths); + break; + } + } + return IdLoader.createTsIdLoader(indexRouting, routingPaths); } else { return IdLoader.fromLeafStoredFieldLoader(); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java index 0e4945f7faea8..329d8a795732f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.index.mapper; +import org.elasticsearch.common.Explicit; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentParser; @@ -16,6 +17,7 @@ import java.io.IOException; import java.util.List; +import java.util.Map; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; @@ -59,4 +61,35 @@ public XContentParser parser() { assertEquals(fieldname, dynamicMappers.get(0).name()); assertEquals(expectedType, dynamicMappers.get(0).typeName()); } + + public void testCreateDynamicStringFieldAsKeywordForDimension() throws IOException { + String source = "{\"f1\": \"foobar\"}"; + XContentParser parser = createParser(JsonXContent.jsonXContent, source); + SourceToParse sourceToParse = new SourceToParse("test", new BytesArray(source), XContentType.JSON); + + SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null).setSynthetic().build(); + RootObjectMapper root = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add( + new PassThroughObjectMapper.Builder("labels").setContainsDimensions().dynamic(ObjectMapper.Dynamic.TRUE) + ).build(MapperBuilderContext.root(false, false)); + Mapping mapping = new Mapping(root, new MetadataFieldMapper[] { sourceMapper }, Map.of()); + + DocumentParserContext ctx = new TestDocumentParserContext(MappingLookup.fromMapping(mapping), sourceToParse) { + @Override + public XContentParser parser() { + return parser; + } + }; + ctx.path().add("labels"); + + // position the parser on the value + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser); + parser.nextToken(); + assertTrue(parser.currentToken().isValue()); + DynamicFieldsBuilder.DYNAMIC_TRUE.createDynamicFieldFromValue(ctx, "f1"); + List dynamicMappers = ctx.getDynamicMappers(); + assertEquals(1, dynamicMappers.size()); + assertEquals("labels.f1", dynamicMappers.get(0).name()); + assertEquals("keyword", dynamicMappers.get(0).typeName()); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java index 4a29dce00436a..6df9fd1f35f52 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java @@ -159,7 +159,7 @@ public void testFieldAliasWithDifferentNestedScopes() { private static FieldMapper createFieldMapper(String parent, String name) { return new BooleanFieldMapper.Builder(name, ScriptCompiler.NONE, false, IndexVersion.current()).build( - new MapperBuilderContext(parent, false, false) + new MapperBuilderContext(parent) ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java index 8eb824884a591..0737dcb7cb5d2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java @@ -322,7 +322,7 @@ private static RootObjectMapper createRootSubobjectFalseLeafWithDots() { private static ObjectMapper.Builder createObjectSubobjectsFalseLeafWithDots() { KeywordFieldMapper.Builder fieldBuilder = new KeywordFieldMapper.Builder("host.name", IndexVersion.current()); - KeywordFieldMapper fieldMapper = fieldBuilder.build(new MapperBuilderContext("foo.metrics", false, false)); + KeywordFieldMapper fieldMapper = fieldBuilder.build(new MapperBuilderContext("foo.metrics")); assertEquals("host.name", fieldMapper.simpleName()); assertEquals("foo.metrics.host.name", fieldMapper.name()); return new ObjectMapper.Builder("foo", ObjectMapper.Defaults.SUBOBJECTS).add( @@ -332,7 +332,7 @@ private static ObjectMapper.Builder createObjectSubobjectsFalseLeafWithDots() { private ObjectMapper.Builder createObjectSubobjectsFalseLeafWithMultiField() { TextFieldMapper.Builder fieldBuilder = createTextKeywordMultiField("host.name"); - TextFieldMapper textKeywordMultiField = fieldBuilder.build(new MapperBuilderContext("foo.metrics", false, false)); + TextFieldMapper textKeywordMultiField = fieldBuilder.build(new MapperBuilderContext("foo.metrics")); assertEquals("host.name", textKeywordMultiField.simpleName()); assertEquals("foo.metrics.host.name", textKeywordMultiField.name()); FieldMapper fieldMapper = textKeywordMultiField.multiFields.iterator().next(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/PassThroughObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/PassThroughObjectMapperTests.java new file mode 100644 index 0000000000000..40994e2835e2b --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/PassThroughObjectMapperTests.java @@ -0,0 +1,135 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import java.io.IOException; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + +public class PassThroughObjectMapperTests extends MapperServiceTestCase { + + public void testSimpleKeyword() throws IOException { + MapperService mapperService = createMapperService(mapping(b -> { + b.startObject("labels").field("type", "passthrough"); + { + b.startObject("properties"); + b.startObject("dim").field("type", "keyword").endObject(); + b.endObject(); + } + b.endObject(); + })); + Mapper mapper = mapperService.mappingLookup().getMapper("labels.dim"); + assertThat(mapper, instanceOf(KeywordFieldMapper.class)); + assertFalse(((KeywordFieldMapper) mapper).fieldType().isDimension()); + } + + public void testKeywordDimension() throws IOException { + MapperService mapperService = createMapperService(mapping(b -> { + b.startObject("labels").field("type", "passthrough").field("time_series_dimension", "true"); + { + b.startObject("properties"); + b.startObject("dim").field("type", "keyword").endObject(); + b.endObject(); + } + b.endObject(); + })); + Mapper mapper = mapperService.mappingLookup().getMapper("labels.dim"); + assertThat(mapper, instanceOf(KeywordFieldMapper.class)); + assertTrue(((KeywordFieldMapper) mapper).fieldType().isDimension()); + } + + public void testDynamic() throws IOException { + MapperService mapperService = createMapperService(mapping(b -> { + b.startObject("labels").field("type", "passthrough").field("dynamic", "true"); + { + b.startObject("properties"); + b.startObject("dim").field("type", "keyword").endObject(); + b.endObject(); + } + b.endObject(); + })); + PassThroughObjectMapper mapper = (PassThroughObjectMapper) mapperService.mappingLookup().objectMappers().get("labels"); + assertEquals(ObjectMapper.Dynamic.TRUE, mapper.dynamic()); + } + + public void testEnabled() throws IOException { + MapperService mapperService = createMapperService(mapping(b -> { + b.startObject("labels").field("type", "passthrough").field("enabled", "false"); + { + b.startObject("properties"); + b.startObject("dim").field("type", "keyword").endObject(); + b.endObject(); + } + b.endObject(); + })); + PassThroughObjectMapper mapper = (PassThroughObjectMapper) mapperService.mappingLookup().objectMappers().get("labels"); + assertEquals(false, mapper.isEnabled()); + } + + public void testSubobjectsThrows() throws IOException { + MapperException exception = expectThrows(MapperException.class, () -> createMapperService(mapping(b -> { + b.startObject("labels").field("type", "passthrough").field("subobjects", "true"); + { + b.startObject("properties"); + b.startObject("dim").field("type", "keyword").endObject(); + b.endObject(); + } + b.endObject(); + }))); + + assertEquals( + "Failed to parse mapping: Mapping definition for [labels] has unsupported parameters: [subobjects : true]", + exception.getMessage() + ); + } + + public void testAddSubobjectThrows() throws IOException { + MapperException exception = expectThrows(MapperException.class, () -> createMapperService(mapping(b -> { + b.startObject("labels").field("type", "passthrough"); + { + b.startObject("properties"); + { + b.startObject("subobj").field("type", "object"); + { + b.startObject("properties"); + b.startObject("dim").field("type", "keyword").endObject(); + b.endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endObject(); + }))); + + assertEquals( + "Failed to parse mapping: Tried to add subobject [subobj] to object [labels] which does not support subobjects", + exception.getMessage() + ); + } + + public void testWithoutMappers() throws IOException { + MapperService mapperService = createMapperService(mapping(b -> { + b.startObject("labels").field("type", "passthrough"); + { + b.startObject("properties"); + b.startObject("dim").field("type", "keyword").endObject(); + b.endObject(); + } + b.endObject(); + b.startObject("shallow").field("type", "passthrough"); + b.endObject(); + })); + + var labels = mapperService.mappingLookup().objectMappers().get("labels"); + var shallow = mapperService.mappingLookup().objectMappers().get("shallow"); + assertThat(labels.withoutMappers().toString(), equalTo(shallow.toString().replace("shallow", "labels"))); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java index b2a6651142181..b77019806fc4f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java @@ -8,9 +8,11 @@ package org.elasticsearch.index.mapper; +import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; @@ -18,6 +20,8 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -342,6 +346,66 @@ public void testRuntimeSectionRemainingField() throws IOException { assertEquals("Failed to parse mapping: unknown parameter [unsupported] on runtime field [field] of type [keyword]", e.getMessage()); } + public void testPassThroughObjectWithAliases() throws IOException { + MapperService mapperService = createMapperService(mapping(b -> { + b.startObject("labels").field("type", "passthrough"); + { + b.startObject("properties"); + b.startObject("dim").field("type", "keyword").endObject(); + b.endObject(); + } + b.endObject(); + })); + assertThat(mapperService.mappingLookup().getMapper("dim"), instanceOf(FieldAliasMapper.class)); + assertThat(mapperService.mappingLookup().getMapper("labels.dim"), instanceOf(KeywordFieldMapper.class)); + } + + public void testAliasMappersCreatesAlias() throws Exception { + var context = MapperBuilderContext.root(false, false); + + Map fields = new HashMap<>(); + fields.put("host", new KeywordFieldMapper.Builder("host", IndexVersion.current()).build(context)); + + Map mappers = new HashMap<>(); + mappers.put( + "labels", + new PassThroughObjectMapper("labels", Explicit.EXPLICIT_TRUE, ObjectMapper.Dynamic.FALSE, fields, Explicit.EXPLICIT_FALSE) + ); + + Map aliases = new RootObjectMapper.Builder("root", Explicit.EXPLICIT_FALSE).getAliasMappers(mappers, context); + assertEquals(1, aliases.size()); + assertThat(aliases.get("host"), instanceOf(FieldAliasMapper.class)); + } + + public void testAliasMappersCreatesNoAliasForRegularObject() throws Exception { + var context = MapperBuilderContext.root(false, false); + + Map fields = new HashMap<>(); + fields.put("host", new KeywordFieldMapper.Builder("host", IndexVersion.current()).build(context)); + + Map mappers = new HashMap<>(); + mappers.put( + "labels", + new ObjectMapper("labels", "labels", Explicit.EXPLICIT_TRUE, Explicit.EXPLICIT_FALSE, ObjectMapper.Dynamic.FALSE, fields) + ); + assertTrue(new RootObjectMapper.Builder("root", Explicit.EXPLICIT_FALSE).getAliasMappers(mappers, context).isEmpty()); + } + + public void testAliasMappersConflictingField() throws Exception { + var context = MapperBuilderContext.root(false, false); + + Map fields = new HashMap<>(); + fields.put("host", new KeywordFieldMapper.Builder("host", IndexVersion.current()).build(context)); + + Map mappers = new HashMap<>(); + mappers.put( + "labels", + new PassThroughObjectMapper("labels", Explicit.EXPLICIT_TRUE, ObjectMapper.Dynamic.FALSE, fields, Explicit.EXPLICIT_FALSE) + ); + mappers.put("host", new KeywordFieldMapper.Builder("host", IndexVersion.current()).build(context)); + assertTrue(new RootObjectMapper.Builder("root", Explicit.EXPLICIT_FALSE).getAliasMappers(mappers, context).isEmpty()); + } + public void testEmptyType() throws Exception { String mapping = Strings.toString( XContentFactory.jsonBuilder() diff --git a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java index 9aa358123d282..477203df8f076 100644 --- a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java +++ b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java @@ -43,6 +43,7 @@ import org.elasticsearch.index.mapper.IdLoader; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.MapperServiceTestCase; import org.elasticsearch.index.mapper.MockFieldMapper; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.ParsedQuery; @@ -65,9 +66,9 @@ import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortAndFormats; import org.elasticsearch.search.sort.SortBuilders; -import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; import java.util.UUID; @@ -87,7 +88,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class DefaultSearchContextTests extends ESTestCase { +public class DefaultSearchContextTests extends MapperServiceTestCase { public void testPreProcess() throws Exception { TimeValue timeout = new TimeValue(randomIntBetween(1, 100)); @@ -475,6 +476,30 @@ public void testNewIdLoaderWithTsdb() throws Exception { } } + public void testNewIdLoaderWithTsdbAndRoutingPathMatch() throws Exception { + Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2000-01-01T00:00:00.000Z") + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2001-01-01T00:00:00.000Z") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "labels.*") + .build(); + + XContentBuilder mappings = mapping(b -> { + b.startObject("labels").field("type", "object"); + { + b.startObject("properties"); + b.startObject("dim").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.endObject(); + } + b.endObject(); + }); + + try (DefaultSearchContext context = createDefaultSearchContext(settings, mappings)) { + assertThat(context.newIdLoader(), instanceOf(IdLoader.TsIdLoader.class)); + context.indexShard().getThreadPool().shutdown(); + } + } + public void testDetermineMaximumNumberOfSlices() { IndexShard indexShard = mock(IndexShard.class); when(indexShard.shardId()).thenReturn(new ShardId("index", "uuid", 0)); @@ -791,6 +816,10 @@ public void testGetFieldCardinalityRuntimeField() { } private DefaultSearchContext createDefaultSearchContext(Settings providedIndexSettings) throws IOException { + return createDefaultSearchContext(providedIndexSettings, null); + } + + private DefaultSearchContext createDefaultSearchContext(Settings providedIndexSettings, XContentBuilder mappings) throws IOException { TimeValue timeout = new TimeValue(randomIntBetween(1, 100)); ShardSearchRequest shardSearchRequest = mock(ShardSearchRequest.class); when(shardSearchRequest.searchType()).thenReturn(SearchType.DEFAULT); @@ -813,15 +842,23 @@ private DefaultSearchContext createDefaultSearchContext(Settings providedIndexSe SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); when(indexService.newSearchExecutionContext(eq(shardId.id()), eq(shardId.id()), any(), any(), nullable(String.class), any())) .thenReturn(searchExecutionContext); - MapperService mapperService = mock(MapperService.class); - when(mapperService.hasNested()).thenReturn(randomBoolean()); - when(indexService.mapperService()).thenReturn(mapperService); IndexMetadata indexMetadata = IndexMetadata.builder("index").settings(settings).build(); - IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + IndexSettings indexSettings; + MapperService mapperService; + if (mappings != null) { + mapperService = createMapperService(settings, mappings); + indexSettings = mapperService.getIndexSettings(); + } else { + indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + mapperService = mock(MapperService.class); + when(mapperService.hasNested()).thenReturn(randomBoolean()); + when(indexService.mapperService()).thenReturn(mapperService); + when(mapperService.getIndexSettings()).thenReturn(indexSettings); + } when(indexService.getIndexSettings()).thenReturn(indexSettings); + when(indexService.mapperService()).thenReturn(mapperService); when(indexService.getMetadata()).thenReturn(indexMetadata); - when(mapperService.getIndexSettings()).thenReturn(indexSettings); when(searchExecutionContext.getIndexSettings()).thenReturn(indexSettings); when(searchExecutionContext.indexVersionCreated()).thenReturn(indexSettings.getIndexVersionCreated()); diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index be26eabe308ba..bee028fa84382 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -101,6 +101,7 @@ import org.elasticsearch.index.mapper.NestedObjectMapper; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.ObjectMapper; +import org.elasticsearch.index.mapper.PassThroughObjectMapper; import org.elasticsearch.index.mapper.RangeFieldMapper; import org.elasticsearch.index.mapper.RangeType; import org.elasticsearch.index.mapper.TextFieldMapper; @@ -200,6 +201,7 @@ public abstract class AggregatorTestCase extends ESTestCase { SparseVectorFieldMapper.CONTENT_TYPE, // Sparse vectors are no longer supported NestedObjectMapper.CONTENT_TYPE, // TODO support for nested + PassThroughObjectMapper.CONTENT_TYPE, // TODO support for passthrough CompletionFieldMapper.CONTENT_TYPE, // TODO support completion FieldAliasMapper.CONTENT_TYPE // TODO support alias ); diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index e8fdf7e0205da..eba2052b9a70b 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -195,6 +195,9 @@ Number parsedNullValue() { @Override public UnsignedLongFieldMapper build(MapperBuilderContext context) { + if (context.parentObjectContainsDimensions()) { + dimension.setValue(true); + } UnsignedLongFieldType fieldType = new UnsignedLongFieldType( context.buildFullName(name), indexed.getValue(),