From ffee9e8833415d4cbdd02280b2b5ed8da63f3680 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 16 Aug 2016 12:57:54 +0200 Subject: [PATCH 1/2] Automatically upgrade analyzed string fields that have `index_options` or `position_increment_gap` set. #20002 Closes #19974 --- .../index/mapper/StringFieldMapper.java | 13 ++++---- .../mapper/StringMappingUpgradeTests.java | 32 ++++++++++++++++++- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/mapper/StringFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/StringFieldMapper.java index 6f529c82ea119..887a40fe70bb5 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/StringFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/StringFieldMapper.java @@ -22,13 +22,9 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.index.IndexOptions; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.RegexpQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; -import org.elasticsearch.common.Nullable; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; @@ -40,7 +36,6 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData; import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData; -import org.elasticsearch.index.query.QueryShardContext; import java.io.IOException; import java.util.Arrays; @@ -71,7 +66,8 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc "type", // common text parameters, for which the upgrade is straightforward "index", "store", "doc_values", "omit_norms", "norms", "fields", "copy_to", - "fielddata", "include_in_all", "analyzer", "search_analyzer", "search_quote_analyzer")); + "fielddata", "include_in_all", "analyzer", "search_analyzer", "search_quote_analyzer", + "index_options", "position_increment_gap")); public static class Defaults { public static double FIELDDATA_MIN_FREQUENCY = 0; @@ -259,7 +255,10 @@ public Mapper.Builder parse(String fieldName, Map node, ParserCo } } - throw new IllegalArgumentException("The [string] type is removed in 5.0. You should now use either a [text] " + Set unsupportedParameters = new HashSet<>(node.keySet()); + unsupportedParameters.removeAll(autoUpgradeParameters); + throw new IllegalArgumentException("The [string] type is removed in 5.0 and automatic upgrade failed because parameters " + + unsupportedParameters + " are not supported for automatic upgrades. You should now use either a [text] " + "or [keyword] field instead for field [" + fieldName + "]"); } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/StringMappingUpgradeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/StringMappingUpgradeTests.java index 4107749376177..311bf0205ed40 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/StringMappingUpgradeTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/StringMappingUpgradeTests.java @@ -106,6 +106,32 @@ public void testUpgradeNotIndexedString() throws IOException { assertEquals(IndexOptions.NONE, field.fieldType().indexOptions()); } + public void testUpgradeIndexOptions() throws IOException { + IndexService indexService = createIndex("test"); + DocumentMapperParser parser = indexService.mapperService().documentMapperParser(); + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties").startObject("field").field("type", "string") + .field("index_options", "offsets").endObject().endObject() + .endObject().endObject().string(); + DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); + FieldMapper field = mapper.mappers().getMapper("field"); + assertThat(field, instanceOf(TextFieldMapper.class)); + assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, field.fieldType().indexOptions()); + } + + public void testUpgradePositionGap() throws IOException { + IndexService indexService = createIndex("test"); + DocumentMapperParser parser = indexService.mapperService().documentMapperParser(); + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties").startObject("field").field("type", "string") + .field("position_increment_gap", 42).endObject().endObject() + .endObject().endObject().string(); + DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); + FieldMapper field = mapper.mappers().getMapper("field"); + assertThat(field, instanceOf(TextFieldMapper.class)); + assertEquals(42, field.fieldType().indexAnalyzer().getPositionIncrementGap("field")); + } + public void testIllegalIndexValue() throws IOException { IndexService indexService = createIndex("test"); DocumentMapperParser parser = indexService.mapperService().documentMapperParser(); @@ -297,7 +323,11 @@ private void doTestUpgradeRandomMapping(int iter) throws IOException { } if (randomBoolean()) { // this option is not upgraded automatically - mapping.field("index_options", "docs"); + if (keyword) { + mapping.field("index_options", "docs"); + } else { + mapping.field("ignore_above", 30); + } shouldUpgrade = false; } mapping.endObject().endObject().endObject().endObject(); From d894db1590ead700d3e3d79c483aba00efa90a33 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 16 Aug 2016 12:20:07 +0200 Subject: [PATCH 2/2] Only use `PUT` for index creation, not POST. #20001 Currently both `PUT` and `POST` can be used to create indices. This commit removes support for `POST index_name` so that we can use it to index documents with auto-generated ids once types are removed. Relates #15613 --- .../admin/indices/RestCreateIndexAction.java | 1 - docs/plugins/mapper-attachments.asciidoc | 2 +- docs/reference/indices/create-index.asciidoc | 56 +++++++++---------- .../migration/migrate_5_0/rest.asciidoc | 5 ++ .../search/suggesters/phrase-suggest.asciidoc | 2 +- .../rest-api-spec/api/indices.create.json | 2 +- 6 files changed, 36 insertions(+), 32 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java index 524cd5feb26a0..2a7f2a629a7f2 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java @@ -43,7 +43,6 @@ public class RestCreateIndexAction extends BaseRestHandler { public RestCreateIndexAction(Settings settings, RestController controller) { super(settings); controller.registerHandler(RestRequest.Method.PUT, "/{index}", this); - controller.registerHandler(RestRequest.Method.POST, "/{index}", this); } @SuppressWarnings({"unchecked"}) diff --git a/docs/plugins/mapper-attachments.asciidoc b/docs/plugins/mapper-attachments.asciidoc index a35d9efe61435..b5ceef39ae817 100644 --- a/docs/plugins/mapper-attachments.asciidoc +++ b/docs/plugins/mapper-attachments.asciidoc @@ -45,7 +45,7 @@ Create a property mapping using the new type `attachment`: [source,js] -------------------------- -POST /trying-out-mapper-attachments +PUT /trying-out-mapper-attachments { "mappings": { "person": { diff --git a/docs/reference/indices/create-index.asciidoc b/docs/reference/indices/create-index.asciidoc index bfd2ba2b791b8..a04fe0ca3f695 100644 --- a/docs/reference/indices/create-index.asciidoc +++ b/docs/reference/indices/create-index.asciidoc @@ -14,15 +14,17 @@ associated with it. [source,js] -------------------------------------------------- -$ curl -XPUT 'http://localhost:9200/twitter/' -d '{ +PUT twitter +{ "settings" : { "index" : { "number_of_shards" : 3, <1> "number_of_replicas" : 2 <2> } } -}' +} -------------------------------------------------- +// CONSOLE <1> Default for `number_of_shards` is 5 <2> Default for `number_of_replicas` is 1 (ie one replica for each primary shard) @@ -33,27 +35,31 @@ index settings can also be defined with http://www.json.org[JSON]: [source,js] -------------------------------------------------- -$ curl -XPUT 'http://localhost:9200/twitter/' -d '{ +PUT twitter +{ "settings" : { "index" : { "number_of_shards" : 3, "number_of_replicas" : 2 } } -}' +} -------------------------------------------------- +// CONSOLE or more simplified [source,js] -------------------------------------------------- -$ curl -XPUT 'http://localhost:9200/twitter/' -d '{ +PUT twitter +{ "settings" : { "number_of_shards" : 3, "number_of_replicas" : 2 } -}' +} -------------------------------------------------- +// CONSOLE [NOTE] You do not have to explicitly specify `index` section inside the @@ -72,7 +78,8 @@ The create index API allows to provide a set of one or more mappings: [source,js] -------------------------------------------------- -curl -XPOST localhost:9200/test -d '{ +PUT test +{ "settings" : { "number_of_shards" : 1 }, @@ -83,8 +90,9 @@ curl -XPOST localhost:9200/test -d '{ } } } -}' +} -------------------------------------------------- +// CONSOLE [float] [[create-index-aliases]] @@ -94,7 +102,8 @@ The create index API allows also to provide a set of <> [source,js] -------------------------------------------------- -curl -XPUT localhost:9200/test -d '{ +PUT test +{ "aliases" : { "alias_1" : {}, "alias_2" : { @@ -104,24 +113,9 @@ curl -XPUT localhost:9200/test -d '{ "routing" : "kimchy" } } -}' --------------------------------------------------- - -[float] -=== Creation Date - -When an index is created, a timestamp is stored in the index metadata for the creation date. By -default this is automatically generated but it can also be specified using the -`creation_date` parameter on the create index API: - -[source,js] --------------------------------------------------- -curl -XPUT localhost:9200/test -d '{ - "creation_date" : 1407751337000 <1> -}' +} -------------------------------------------------- - -<1> `creation_date` is set using epoch time in milliseconds. +// CONSOLE [float] [[create-index-wait-for-active-shards]] @@ -138,6 +132,7 @@ what happened: "shards_acknowledged": true } -------------------------------------------------- +// TESTRESPONSE `acknowledged` indicates whether the index was successfully created in the cluster, while `shards_acknowledged` indices whether the requisite number of shard copies were started for @@ -156,19 +151,24 @@ the `wait_for_active_shards` value on all subsequent write operations): [source,js] -------------------------------------------------- -curl -XPUT localhost:9200/test -d '{ +PUT test +{ "settings": { "index.write.wait_for_active_shards": "2" } } -------------------------------------------------- +// CONSOLE +// TEST[skip:requires two nodes] or through the request parameter `wait_for_active_shards`: [source,js] -------------------------------------------------- -curl -XPUT localhost:9200/test?wait_for_active_shards=2 +PUT test?wait_for_active_shards=2 -------------------------------------------------- +// CONSOLE +// TEST[skip:requires two nodes] A detailed explanation of `wait_for_active_shards` and its possible values can be found <>. diff --git a/docs/reference/migration/migrate_5_0/rest.asciidoc b/docs/reference/migration/migrate_5_0/rest.asciidoc index 9d135c4d5bf9a..36f4071ce1587 100644 --- a/docs/reference/migration/migrate_5_0/rest.asciidoc +++ b/docs/reference/migration/migrate_5_0/rest.asciidoc @@ -15,6 +15,11 @@ endpoint should be used in lieu of optimize. The `GET` HTTP verb for `/_forcemerge` is no longer supported, please use the `POST` HTTP verb. +==== Index creation endpoint only accepts `PUT` + +It used to be possible to create an index by either calling `PUT index_name` +or `POST index_name`. Only the former is now supported. + ==== Removed `mem` section from `/_cluster/stats` response The `mem` section contained only one value, the total memory available diff --git a/docs/reference/search/suggesters/phrase-suggest.asciidoc b/docs/reference/search/suggesters/phrase-suggest.asciidoc index 487075a5677f8..03cbb5af35cd9 100644 --- a/docs/reference/search/suggesters/phrase-suggest.asciidoc +++ b/docs/reference/search/suggesters/phrase-suggest.asciidoc @@ -23,7 +23,7 @@ work. The `reverse` analyzer is used only in the last example. [source,js] -------------------------------------------------- -POST test +PUT test { "settings": { "index": { diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create.json index b0267eae3a3c7..1433c893e251e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create.json @@ -1,7 +1,7 @@ { "indices.create": { "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-index.html", - "methods": ["PUT", "POST"], + "methods": ["PUT"], "url": { "path": "/{index}", "paths": ["/{index}"],