diff --git a/docs/reference/mapping/removal_of_types.asciidoc b/docs/reference/mapping/removal_of_types.asciidoc index acfed6874a426..67ef215931b48 100644 --- a/docs/reference/mapping/removal_of_types.asciidoc +++ b/docs/reference/mapping/removal_of_types.asciidoc @@ -424,3 +424,76 @@ POST _reindex ---- // NOTCONSOLE +[float] +=== Index templates + +It is recommended to make index templates typeless before upgrading to 7.0 by +re-adding them with `include_type_name` set to `false`. + +In case typeless templates are used with typed index creation calls or typed +templates are used with typeless index creation calls, the template will still +be applied but the index creation call decides whether there should be a type +or not. For instance in the below example, `index-1-01` will have a type in +spite of the fact that it matches a template that is typeless, and `index-2-01` +will be typeless in spite of the fact that it matches a template that defines +a type. Both `index-1-01` and `index-2-01` will inherit the `foo` field from +the template that they match. + +[source,js] +-------------------------------------------------- +PUT _template/template1?include_type_name=false +{ + "index_patterns":[ "index-1-*" ], + "mappings": { + "properties": { + "foo": { + "type": "keyword" + } + } + } +} + +PUT _template/template2?include_type_name=true +{ + "index_patterns":[ "index-2-*" ], + "mappings": { + "type": { + "properties": { + "foo": { + "type": "keyword" + } + } + } + } +} + +PUT index-1-01?include_type_name=true +{ + "mappings": { + "type": { + "properties": { + "bar": { + "type": "long" + } + } + } + } +} + +PUT index-2-01?include_type_name=false +{ + "mappings": { + "properties": { + "bar": { + "type": "long" + } + } + } +} +-------------------------------------------------- +// CONSOLE + +In case of implicit index creation, because of documents that get indexed in +an index that doesn't exist yet, the template is always honored. This is +usually not a problem due to the fact that typless index calls work on typed +indices. diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/20_mix_typeless_typeful.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/20_mix_typeless_typeful.yml new file mode 100644 index 0000000000000..e443ed1393594 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/20_mix_typeless_typeful.yml @@ -0,0 +1,129 @@ +--- +"Create a typeless index while there is a typed template": + + - skip: + version: " - 6.6.99" + reason: Merging typeless/typed mappings/templates was added in 6.7 + + - do: + indices.put_template: + include_type_name: true + name: test_template + body: + index_patterns: test-* + mappings: + my_type: + properties: + foo: + type: keyword + + - do: + indices.create: + include_type_name: false + index: test-1 + body: + mappings: + properties: + bar: + type: "long" + + - do: + indices.get_mapping: + include_type_name: true + index: test-1 + + - is_true: test-1.mappings._doc # the index creation call won + - is_false: test-1.mappings.my_type + - is_true: test-1.mappings._doc.properties.foo + - is_true: test-1.mappings._doc.properties.bar + +--- +"Create a typed index while there is a typeless template": + + - skip: + version: " - 6.6.99" + reason: Merging typeless/typed mappings/templates was added in 6.7 + + - do: + indices.put_template: + include_type_name: false + name: test_template + body: + index_patterns: test-* + mappings: + properties: + foo: + type: keyword + + - do: + indices.create: + include_type_name: true + index: test-1 + body: + mappings: + my_type: + properties: + bar: + type: "long" + + - do: + indices.get_mapping: + include_type_name: true + index: test-1 + + - is_true: test-1.mappings.my_type # the index creation call won + - is_false: test-1.mappings._doc + - is_true: test-1.mappings.my_type.properties.foo + - is_true: test-1.mappings.my_type.properties.bar + +--- +"Implicitly create a typed index while there is a typeless template": + + - skip: + version: " - 6.6.99" + reason: include_type_name only supported as of 6.7 + + - do: + indices.put_template: + include_type_name: false + name: test_template + body: + index_patterns: test-* + mappings: + properties: + foo: + type: keyword + + - do: + catch: /the final mapping would have more than 1 type/ + index: + index: test-1 + type: my_type + body: { bar: 42 } + +--- +"Implicitly create a typeless index while there is a typed template": + + - skip: + version: " - 6.6.99" + reason: include_type_name only supported as of 6.7 + + - do: + indices.put_template: + include_type_name: true + name: test_template + body: + index_patterns: test-* + mappings: + my_type: + properties: + foo: + type: keyword + + - do: + catch: /the final mapping would have more than 1 type/ + index: + index: test-1 + type: _doc + body: { bar: 42 } + diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/20_mix_typeless_typeful.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/20_mix_typeless_typeful.yml index c3b57c4cabdfe..14f13bb10dbe2 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/20_mix_typeless_typeful.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/20_mix_typeless_typeful.yml @@ -42,11 +42,39 @@ type: "keyword" # also test no-op updates that trigger special logic wrt the mapping version - do: - catch: bad_request + catch: /the final mapping would have more than 1 type/ indices.put_mapping: index: index + type: some_other_type body: some_other_type: properties: bar: type: "long" + + +--- +"PUT mapping with _doc on an index that has types": + + - do: + indices.create: + include_type_name: true + index: index + body: + mappings: + my_type: + properties: + foo: + type: "keyword" + + - do: + catch: /the final mapping would have more than 1 type/ + indices.put_mapping: + include_type_name: true + index: index + type: _doc + body: + _doc: + properties: + bar: + type: "long" diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 6ba0c9e5da12a..a81f35a1c622f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -321,6 +321,28 @@ public ClusterState execute(ClusterState currentState) throws Exception { if (mappings.containsKey(cursor.key)) { XContentHelper.mergeDefaults(mappings.get(cursor.key), MapperService.parseMapping(xContentRegistry, mappingString)); + } else if (mappings.size() == 1 && cursor.key.equals(MapperService.SINGLE_MAPPING_NAME)) { + // Typeless template with typed mapping + Map templateMapping = MapperService.parseMapping(xContentRegistry, mappingString); + assert templateMapping.size() == 1 : templateMapping; + assert cursor.key.equals(templateMapping.keySet().iterator().next()) : + cursor.key + " != " + templateMapping; + Map.Entry> mappingEntry = mappings.entrySet().iterator().next(); + templateMapping = Collections.singletonMap( + mappingEntry.getKey(), // reuse type name from the mapping + templateMapping.values().iterator().next()); // but actual mappings from the template + XContentHelper.mergeDefaults(mappingEntry.getValue(), templateMapping); + } else if (template.mappings().size() == 1 && mappings.containsKey(MapperService.SINGLE_MAPPING_NAME)) { + // Typed template with typeless mapping + Map templateMapping = MapperService.parseMapping(xContentRegistry, mappingString); + assert templateMapping.size() == 1 : templateMapping; + assert cursor.key.equals(templateMapping.keySet().iterator().next()) : + cursor.key + " != " + templateMapping; + Map mapping = mappings.get(MapperService.SINGLE_MAPPING_NAME); + templateMapping = Collections.singletonMap( + MapperService.SINGLE_MAPPING_NAME, // make template mapping typeless + templateMapping.values().iterator().next()); + XContentHelper.mergeDefaults(mapping, templateMapping); } else { mappings.put(cursor.key, MapperService.parseMapping(xContentRegistry, mappingString)); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java index c035c73a42e10..0c04daf0e4bd7 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java @@ -38,6 +38,8 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; @@ -273,7 +275,10 @@ private ClusterState applyRequest(ClusterState currentState, PutMappingClusterSt updateList.add(indexMetaData); // try and parse it (no need to add it here) so we can bail early in case of parsing exception DocumentMapper newMapper; - DocumentMapper existingMapper = getMapperForUpdate(mapperService, mappingType); + DocumentMapper existingMapper = mapperService.documentMapper(mappingType); + if (existingMapper == null && isMappingSourceTyped(mapperService, mappingUpdateSource, request.type()) == false) { + existingMapper = getMapperForUpdate(mapperService, mappingType); + } String typeForUpdate = existingMapper == null ? mappingType : existingMapper.type(); if (MapperService.DEFAULT_MAPPING.equals(typeForUpdate)) { @@ -325,9 +330,16 @@ private ClusterState applyRequest(ClusterState currentState, PutMappingClusterSt // we use the exact same indexService and metadata we used to validate above here to actually apply the update final Index index = indexMetaData.getIndex(); final MapperService mapperService = indexMapperServices.get(index); + + // If the _type name is _doc and there is no _doc top-level key then this means that we + // are handling a typeless call. In such a case, we override _doc with the actual type + // name in the mappings. This allows to use typeless APIs on typed indices. String typeForUpdate = mappingType; CompressedXContent existingSource = null; - DocumentMapper existingMapper = getMapperForUpdate(mapperService, mappingType); + DocumentMapper existingMapper = mapperService.documentMapper(mappingType); + if (existingMapper == null && isMappingSourceTyped(mapperService, mappingUpdateSource, request.type()) == false) { + existingMapper = getMapperForUpdate(mapperService, mappingType); + } if (existingMapper != null) { typeForUpdate = existingMapper.type(); existingSource = existingMapper.mappingSource(); @@ -388,6 +400,15 @@ public String describeTasks(List tasks) { } } + /** + * Returns {@code true} if the given {@code mappingSource} includes a type + * as a top-level object. + */ + private static boolean isMappingSourceTyped(MapperService mapperService, CompressedXContent mappingSource, String type) { + Map root = XContentHelper.convertToMap(mappingSource.compressedReference(), true, XContentType.JSON).v2(); + return root.size() == 1 && root.keySet().iterator().next().equals(type); + } + public void putMapping(final PutMappingClusterStateUpdateRequest request, final ActionListener listener) { clusterService.submitStateUpdateTask("put-mapping", request, diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java index 8a6d46b102989..7019162ad530e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java @@ -348,6 +348,36 @@ public void testWriteIndexValidationException() throws Exception { + "you must manage this on the create index request or with an index template"); } + public void testTypelessTemplateWithTypedIndexCreation() throws Exception { + reqSettings.put(SETTING_NUMBER_OF_SHARDS, 1); + addMatchingTemplate(builder -> builder.putMapping("type", "{\"type\": {}}")); + setupRequestMapping(MapperService.SINGLE_MAPPING_NAME, new CompressedXContent("{\"_doc\":{}}")); + executeTask(); + assertThat(getMappingsFromResponse(), Matchers.hasKey(MapperService.SINGLE_MAPPING_NAME)); + } + + public void testTypedTemplateWithTypelessIndexCreation() throws Exception { + reqSettings.put(SETTING_NUMBER_OF_SHARDS, 1); + addMatchingTemplate(builder -> builder.putMapping(MapperService.SINGLE_MAPPING_NAME, "{\"_doc\": {}}")); + setupRequestMapping("type", new CompressedXContent("{\"type\":{}}")); + executeTask(); + assertThat(getMappingsFromResponse(), Matchers.hasKey("type")); + } + + public void testTypedTemplate() throws Exception { + reqSettings.put(SETTING_NUMBER_OF_SHARDS, 1); + addMatchingTemplate(builder -> builder.putMapping("type", "{\"type\": {}}")); + executeTask(); + assertThat(getMappingsFromResponse(), Matchers.hasKey("type")); + } + + public void testTypelessTemplate() throws Exception { + reqSettings.put(SETTING_NUMBER_OF_SHARDS, 1); + addMatchingTemplate(builder -> builder.putMapping(MapperService.SINGLE_MAPPING_NAME, "{\"_doc\": {}}")); + executeTask(); + assertThat(getMappingsFromResponse(), Matchers.hasKey(MapperService.SINGLE_MAPPING_NAME)); + } + private IndexRoutingTable createIndexRoutingTableWithStartedShards(Index index) { final IndexRoutingTable idxRoutingTable = mock(IndexRoutingTable.class);