Merge branch 'master' into byte-size-value-setting-limit-error-message

* master: Build: Fix Java9 MR build (elastic#29312) Reindex: Fix error in delete-by-query rest spec (elastic#29318) Improve similarity integration. (elastic#29187) Fix some query extraction bugs. (elastic#29283) [Docs] Correct experimental note formatting Move Nullable into core (elastic#29341) [Docs] Update getting-started.asciidoc (elastic#29294) Elasticsearch 6.3.0 is now on Lucene 7.3. [DOCS] Refer back to index API for full-document updates in _update API section (elastic#28677) Fix missing comma in ingest-node.asciidoc (elastic#29343) Improve exception handling on TransportMasterNodeAction (elastic#29314) Don't break allocation if resize source index is missing (elastic#29311) Use fixture to test repository-s3 plugin (elastic#29296) Fix NDCG for empty search results (elastic#29267) Pass through script params in scripted metric agg (elastic#29154) Fix Eclipse build. Upgrade to lucene-7.3.0-snapshot-98a6b3d. (elastic#29298) Painless: Remove extraneous INLINE constant. (elastic#29340) Remove HTTP max content length leniency (elastic#29337) Begin moving XContent to a separate lib/artifact (elastic#29300)
jasontedor · Apr 3, 2018 · 3f07930 · 3f07930
2 parents 6165037 + 7c6d5cb
commit 3f07930
Show file tree

Hide file tree

Showing 226 changed files with 2,571 additions and 1,576 deletions.
diff --git a/build.gradle b/build.gradle
@@ -196,6 +196,7 @@ subprojects {
     "org.elasticsearch:elasticsearch-cli:${version}": ':server:cli',
     "org.elasticsearch:elasticsearch-core:${version}": ':libs:elasticsearch-core',
     "org.elasticsearch:elasticsearch-nio:${version}": ':libs:elasticsearch-nio',
+    "org.elasticsearch:elasticsearch-x-content:${version}": ':libs:x-content',
     "org.elasticsearch:elasticsearch-secure-sm:${version}": ':libs:secure-sm',
     "org.elasticsearch.client:elasticsearch-rest-client:${version}": ':client:rest',
     "org.elasticsearch.client:elasticsearch-rest-client-sniffer:${version}": ':client:sniffer',

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -551,7 +551,7 @@ class BuildPlugin implements Plugin<Project> {
                 if (project.licenseFile == null || project.noticeFile == null) {
                     throw new GradleException("Must specify license and notice file for project ${project.path}")
                 }
-                jarTask.into('META-INF') {
+                jarTask.metaInf {
                     from(project.licenseFile.parent) {
                         include project.licenseFile.name
                     }

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
@@ -95,7 +95,7 @@ public class PluginBuildPlugin extends BuildPlugin {
             // we "upgrade" these optional deps to provided for plugins, since they will run
             // with a full elasticsearch server that includes optional deps
             compileOnly "org.locationtech.spatial4j:spatial4j:${project.versions.spatial4j}"
-            compileOnly "com.vividsolutions:jts:${project.versions.jts}"
+            compileOnly "org.locationtech.jts:jts-core:${project.versions.jts}"
             compileOnly "org.apache.logging.log4j:log4j-api:${project.versions.log4j}"
             compileOnly "org.apache.logging.log4j:log4j-core:${project.versions.log4j}"
             compileOnly "org.elasticsearch:jna:${project.versions.jna}"

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.gradle.precommit
 
+import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApis
 import de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin
 import org.gradle.api.Project
 import org.gradle.api.Task
@@ -83,17 +84,14 @@ class PrecommitTasks {
                               getClass().getResource('/forbidden/es-all-signatures.txt')]
             suppressAnnotations = ['**.SuppressForbidden']
         }
-        Task mainForbidden = project.tasks.findByName('forbiddenApisMain')
-        if (mainForbidden != null) {
-            mainForbidden.configure {
-                signaturesURLs += getClass().getResource('/forbidden/es-server-signatures.txt')
-            }
-        }
-        Task testForbidden = project.tasks.findByName('forbiddenApisTest')
-        if (testForbidden != null) {
-            testForbidden.configure {
-                signaturesURLs += getClass().getResource('/forbidden/es-test-signatures.txt')
-                signaturesURLs += getClass().getResource('/forbidden/http-signatures.txt')
+        project.tasks.withType(CheckForbiddenApis) {
+            // we do not use the += operator to add signatures, as conventionMappings of Gradle do not work when it's configured using withType:
+            if (name.endsWith('Test')) {
+                signaturesURLs = project.forbiddenApis.signaturesURLs +
+                    [ getClass().getResource('/forbidden/es-test-signatures.txt'), getClass().getResource('/forbidden/http-signatures.txt') ]
+            } else {
+                signaturesURLs = project.forbiddenApis.signaturesURLs +
+                    [ getClass().getResource('/forbidden/es-server-signatures.txt') ]
             }
         }
         Task forbiddenApis = project.tasks.findByName('forbiddenApis')
@@ -144,21 +142,15 @@ class PrecommitTasks {
             ]
             toolVersion = 7.5
         }
-        for (String taskName : ['checkstyleMain', 'checkstyleJava9', 'checkstyleTest']) {
-            Task task = project.tasks.findByName(taskName)
-            if (task != null) {
-                project.tasks['check'].dependsOn.remove(task)
-                checkstyleTask.dependsOn(task)
-                task.dependsOn(copyCheckstyleConf)
-                task.inputs.file(checkstyleSuppressions)
-                task.reports {
-                    html.enabled false
-                }
-            }
-        }
 
-        project.tasks.withType(Checkstyle) {
-            dependsOn(copyCheckstyleConf)
+        project.tasks.withType(Checkstyle) { task ->
+            project.tasks[JavaBasePlugin.CHECK_TASK_NAME].dependsOn.remove(task)
+            checkstyleTask.dependsOn(task)
+            task.dependsOn(copyCheckstyleConf)
+            task.inputs.file(checkstyleSuppressions)
+            task.reports {
+                html.enabled false
+            }
         }
 
         return checkstyleTask

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -248,9 +248,7 @@
   <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]concurrent[/\\]EsExecutors.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]concurrent[/\\]ThreadBarrier.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]concurrent[/\\]ThreadContext.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]xcontent[/\\]XContentFactory.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]xcontent[/\\]XContentHelper.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]xcontent[/\\]smile[/\\]SmileXContent.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]Discovery.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]DiscoverySettings.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]ZenDiscovery.java" checks="LineLength" />

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
@@ -1,9 +1,9 @@
 elasticsearch     = 7.0.0-alpha1
-lucene            = 7.2.1
+lucene            = 7.3.0-snapshot-98a6b3d
 
 # optional dependencies
-spatial4j         = 0.6
-jts               = 1.13
+spatial4j         = 0.7
+jts               = 1.15.0
 jackson           = 2.8.10
 snakeyaml         = 1.17
 # when updating log4j, please update also docs/java-api/index.asciidoc

diff --git a/docs/Versions.asciidoc b/docs/Versions.asciidoc
@@ -1,7 +1,7 @@
 :version:               7.0.0-alpha1
 :major-version:         7.x
-:lucene_version:        7.2.1
-:lucene_version_path:   7_2_1
+:lucene_version:        7.3.0
+:lucene_version_path:   7_3_0
 :branch:                master
 :jdk:                   1.8.0_131
 :jdk_major:             8

diff --git a/docs/java-api/query-dsl/geo-shape-query.asciidoc b/docs/java-api/query-dsl/geo-shape-query.asciidoc
@@ -12,13 +12,13 @@ to your classpath in order to use this type:
 <dependency>
     <groupId>org.locationtech.spatial4j</groupId>
     <artifactId>spatial4j</artifactId>
-    <version>0.6</version>                        <1>
+    <version>0.7</version>                        <1>
 </dependency>
 
 <dependency>
-    <groupId>com.vividsolutions</groupId>
-    <artifactId>jts</artifactId>
-    <version>1.13</version>                         <2>
+    <groupId>org.locationtech.jts</groupId>
+    <artifactId>jts-core</artifactId>
+    <version>1.15.0</version>                         <2>
     <exclusions>
         <exclusion>
             <groupId>xerces</groupId>
@@ -28,7 +28,7 @@ to your classpath in order to use this type:
 </dependency>
 -----------------------------------------------
 <1> check for updates in http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.locationtech.spatial4j%22%20AND%20a%3A%22spatial4j%22[Maven Central]
-<2> check for updates in http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22com.vividsolutions%22%20AND%20a%3A%22jts%22[Maven Central]
+<2> check for updates in http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.locationtech.jts%22%20AND%20a%3A%22jts-core%22[Maven Central]
 
 [source,java]
 --------------------------------------------------

diff --git a/docs/reference/docs/update.asciidoc b/docs/reference/docs/update.asciidoc
@@ -118,8 +118,11 @@ POST test/_doc/1/_update
 
 The update API also support passing a partial document,
 which will be merged into the existing document (simple recursive merge,
-inner merging of objects, replacing core "keys/values" and arrays). For
-example:
+inner merging of objects, replacing core "keys/values" and arrays).
+To fully replace the existing document, the <<docs-index_,`index` API>> should
+be used instead.
+The following partial update adds a new field to the
+existing document:
 
 [source,js]
 --------------------------------------------------

diff --git a/docs/reference/getting-started.asciidoc b/docs/reference/getting-started.asciidoc
@@ -777,7 +777,7 @@ GET /bank/_search
 // CONSOLE
 // TEST[continued]
 
-The difference here is that instead of passing `q=*` in the URI, we POST a JSON-style query request body to the `_search` API. We'll discuss this JSON query in the next section.
+The difference here is that instead of passing `q=*` in the URI, we provide a JSON-style query request body to the `_search` API. We'll discuss this JSON query in the next section.
 
 ////
 Hidden response just so we can assert that it is indeed the same but don't have

diff --git a/docs/reference/index-modules/similarity.asciidoc b/docs/reference/index-modules/similarity.asciidoc
@@ -82,20 +82,6 @@ This similarity has the following options:
 
 Type name: `BM25`
 
-[float]
-[[classic-similarity]]
-==== Classic similarity
-
-The classic similarity that is based on the TF/IDF model. This
-similarity has the following option:
-
-`discount_overlaps`::
-    Determines whether overlap tokens (Tokens with
-    0 position increment) are ignored when computing norm. By default this
-    is true, meaning overlap tokens do not count when computing norms.
-
-Type name: `classic`
-
 [float]
 [[dfr]]
 ==== DFR similarity
@@ -541,7 +527,7 @@ PUT /index
     "index": {
       "similarity": {
         "default": {
-          "type": "classic"
+          "type": "boolean"
         }
       }
     }
@@ -563,7 +549,7 @@ PUT /index/_settings
   "index": {
     "similarity": {
       "default": {
-        "type": "classic"
+        "type": "boolean"
       }
     }
   }

diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc
@@ -1341,7 +1341,7 @@ Here is an example of a pipeline specifying custom pattern definitions:
     {
       "grok": {
         "field": "message",
-        "patterns": ["my %{FAVORITE_DOG:dog} is colored %{RGB:color}"]
+        "patterns": ["my %{FAVORITE_DOG:dog} is colored %{RGB:color}"],
         "pattern_definitions" : {
           "FAVORITE_DOG" : "beagle",
           "RGB" : "RED|GREEN|BLUE"

diff --git a/docs/reference/mapping/params/similarity.asciidoc b/docs/reference/mapping/params/similarity.asciidoc
@@ -44,13 +44,9 @@ PUT my_index
         "default_field": { <1>
           "type": "text"
         },
-        "classic_field": {
-          "type": "text",
-          "similarity": "classic" <2>
-        },
         "boolean_sim_field": {
           "type": "text",
-          "similarity": "boolean" <3>
+          "similarity": "boolean" <2>
         }
       }
     }
@@ -59,5 +55,4 @@ PUT my_index
 --------------------------------------------------
 // CONSOLE
 <1> The `default_field` uses the `BM25` similarity.
-<2> The `classic_field` uses the `classic` similarity (ie TF/IDF).
-<3> The `boolean_sim_field` uses the `boolean` similarity.
+<2> The `boolean_sim_field` uses the `boolean` similarity.
diff --git a/docs/reference/migration/migrate_7_0/mappings.asciidoc b/docs/reference/migration/migrate_7_0/mappings.asciidoc
@@ -24,3 +24,16 @@ the index setting `index.mapping.nested_objects.limit`.
 ==== The `update_all_types` option has been removed
 
 This option is useless now that all indices have at most one type.
+
+=== The `classic` similarity has been removed
+
+The `classic` similarity relied on coordination factors for scoring to be good
+in presence of stopwords in the query. This feature has been removed from
+Lucene, which means that the `classic` similarity now produces scores of lower
+quality. It is advised to switch to `BM25` instead, which is widely accepted
+as a better alternative.
+
+=== Similarities fail when unsupported options are provided
+
+An error will now be thrown when unknown configuration options are provided
+to similarities. Such unknown parameters were ignored before.
diff --git a/docs/reference/modules/http.asciidoc b/docs/reference/modules/http.asciidoc
@@ -39,7 +39,7 @@ from the outside. Defaults to the actual port assigned via `http.port`.
 |`http.host` |Used to set the `http.bind_host` and the `http.publish_host` Defaults to `http.host` or `network.host`.
 
 |`http.max_content_length` |The max content of an HTTP request. Defaults to
-`100mb`. If set to greater than `Integer.MAX_VALUE`, it will be reset to 100mb.
+`100mb`.
 
 |`http.max_initial_line_length` |The max length of an HTTP URL. Defaults
 to `4kb`

diff --git a/docs/reference/search/rank-eval.asciidoc b/docs/reference/search/rank-eval.asciidoc
@@ -1,9 +1,7 @@
 [[search-rank-eval]]
 == Ranking Evaluation API
 
-experimental[The ranking evaluation API is experimental and may be changed or removed completely in a future release,
-as well as change in non-backwards compatible ways on minor versions updates. Elastic will take a best effort 
-approach to fix any issues, but experimental features are not subject to the support SLA of official GA features.]
+experimental[The ranking evaluation API is experimental and may be changed or removed completely in a future release, as well as change in non-backwards compatible ways on minor versions updates. Elastic will take a best effort approach to fix any issues, but experimental features are not subject to the support SLA of official GA features.]
 
 The ranking evaluation API allows to evaluate the quality of ranked search
 results over a set of typical search queries. Given this set of queries and a

diff --git a/...va/org/elasticsearch/common/Booleans.java → ...va/org/elasticsearch/common/Booleans.java b/...va/org/elasticsearch/common/Booleans.java → ...va/org/elasticsearch/common/Booleans.java
@@ -73,21 +73,34 @@ public static boolean parseBoolean(String value) {
         throw new IllegalArgumentException("Failed to parse value [" + value + "] as only [true] or [false] are allowed.");
     }
 
+    private static boolean hasText(CharSequence str) {
+        if (str == null || str.length() == 0) {
+            return false;
+        }
+        int strLen = str.length();
+        for (int i = 0; i < strLen; i++) {
+            if (!Character.isWhitespace(str.charAt(i))) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     /**
      *
      * @param value text to parse.
      * @param defaultValue The default value to return if the provided value is <code>null</code>.
      * @return see {@link #parseBoolean(String)}
      */
     public static boolean parseBoolean(String value, boolean defaultValue) {
-        if (Strings.hasText(value)) {
+        if (hasText(value)) {
             return parseBoolean(value);
         }
         return defaultValue;
     }
 
     public static Boolean parseBoolean(String value, Boolean defaultValue) {
-        if (Strings.hasText(value)) {
+        if (hasText(value)) {
             return parseBoolean(value);
         }
         return defaultValue;

diff --git a/...elasticsearch/common/CheckedFunction.java → ...elasticsearch/common/CheckedFunction.java b/...elasticsearch/common/CheckedFunction.java → ...elasticsearch/common/CheckedFunction.java
diff --git a/libs/elasticsearch-core/src/main/java/org/elasticsearch/common/Glob.java b/libs/elasticsearch-core/src/main/java/org/elasticsearch/common/Glob.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common;
+
+/**
+ * Utility class for glob-like matching
+ */
+public class Glob {
+
+    /**
+     * Match a String against the given pattern, supporting the following simple
+     * pattern styles: "xxx*", "*xxx", "*xxx*" and "xxx*yyy" matches (with an
+     * arbitrary number of pattern parts), as well as direct equality.
+     *
+     * @param pattern the pattern to match against
+     * @param str     the String to match
+     * @return whether the String matches the given pattern
+     */
+    public static boolean globMatch(String pattern, String str) {
+        if (pattern == null || str == null) {
+            return false;
+        }
+        int firstIndex = pattern.indexOf('*');
+        if (firstIndex == -1) {
+            return pattern.equals(str);
+        }
+        if (firstIndex == 0) {
+            if (pattern.length() == 1) {
+                return true;
+            }
+            int nextIndex = pattern.indexOf('*', firstIndex + 1);
+            if (nextIndex == -1) {
+                return str.endsWith(pattern.substring(1));
+            } else if (nextIndex == 1) {
+                // Double wildcard "**" - skipping the first "*"
+                return globMatch(pattern.substring(1), str);
+            }
+            String part = pattern.substring(1, nextIndex);
+            int partIndex = str.indexOf(part);
+            while (partIndex != -1) {
+                if (globMatch(pattern.substring(nextIndex), str.substring(partIndex + part.length()))) {
+                    return true;
+                }
+                partIndex = str.indexOf(part, partIndex + 1);
+            }
+            return false;
+        }
+        return (str.length() >= firstIndex &&
+            pattern.substring(0, firstIndex).equals(str.substring(0, firstIndex)) &&
+            globMatch(pattern.substring(firstIndex), str.substring(firstIndex)));
+    }
+
+}
diff --git a/...va/org/elasticsearch/common/Nullable.java → ...va/org/elasticsearch/common/Nullable.java b/...va/org/elasticsearch/common/Nullable.java → ...va/org/elasticsearch/common/Nullable.java