From 891320f5ac5d92a89dbe90f050919d0bd9f671a0 Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Tue, 29 Jan 2019 07:20:09 +0100 Subject: [PATCH 001/100] Elasticsearch support to JSON logging (#36833) In order to support JSON log format, a custom pattern layout was used and its configuration is enclosed in ESJsonLayout. Users are free to use their own patterns, but if smooth Beats integration is needed, they should use ESJsonLayout. EvilLoggerTests are left intact to make sure user's custom log patterns work fine. To populate additional fields node.id and cluster.uuid which are not available at start time, a cluster state update will have to be received and the values passed to log4j pattern converter. A ClusterStateObserver.Listener is used to receive only one ClusteStateUpdate. Once update is received the nodeId and clusterUUid are set in a static field in a NodeAndClusterIdConverter. Following fields are expected in JSON log lines: type, tiemstamp, level, component, cluster.name, node.name, node.id, cluster.uuid, message, stacktrace see ESJsonLayout.java for more details and field descriptions Docker log4j2 configuration is now almost the same as the one use for ES binary. The only difference is that docker is using console appenders, whereas ES is using file appenders. relates: #32850 --- .../archives/integ-test-zip/build.gradle | 2 +- ...sIT.java => JsonLogsFormatAndParseIT.java} | 8 +- .../src/docker/config/log4j2.properties | 44 +++- distribution/src/config/log4j2.properties | 120 +++++++-- docs/reference/migration/migrate_7_0.asciidoc | 2 + .../migration/migrate_7_0/logging.asciidoc | 33 +++ docs/reference/setup/logging-config.asciidoc | 106 ++++++-- qa/die-with-dignity/build.gradle | 2 +- .../qa/die_with_dignity/DieWithDignityIT.java | 78 ++++-- qa/logging-config/build.gradle | 42 ++++ qa/logging-config/custom-log4j2.properties | 31 +++ .../common/logging/JsonLoggerTests.java | 232 ++++++++++++++++++ .../custom_logging/CustomLoggingConfigIT.java | 72 ++++++ .../logging/json_layout/log4j2.properties | 21 ++ .../src/test/resources/plugin-security.policy | 4 + qa/unconfigured-node-name/build.gradle | 2 +- ...sIT.java => JsonLogsFormatAndParseIT.java} | 6 +- .../resources/packaging/tests/60_systemd.bats | 2 +- .../test/resources/packaging/utils/utils.bash | 2 +- .../common/logging/ESJsonLayout.java | 118 +++++++++ .../JsonThrowablePatternConverter.java | 105 ++++++++ .../logging/NodeAndClusterIdConverter.java | 78 ++++++ .../NodeAndClusterIdStateListener.java | 77 ++++++ .../java/org/elasticsearch/node/Node.java | 8 +- .../JsonThrowablePatternConverterTests.java | 93 +++++++ .../common/logging/JsonLogLine.java | 158 ++++++++++++ .../common/logging/JsonLogsIntegTestCase.java | 129 ++++++++++ .../common/logging/JsonLogsStream.java | 97 ++++++++ .../logging/NodeNameInLogsIntegTestCase.java | 101 -------- .../downgrade-to-basic-license/build.gradle | 5 +- .../xpack/ccr/FollowIndexIT.java | 51 ++-- 31 files changed, 1624 insertions(+), 205 deletions(-) rename distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/{NodeNameInLogsIT.java => JsonLogsFormatAndParseIT.java} (88%) create mode 100644 docs/reference/migration/migrate_7_0/logging.asciidoc create mode 100644 qa/logging-config/build.gradle create mode 100644 qa/logging-config/custom-log4j2.properties create mode 100644 qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLoggerTests.java create mode 100644 qa/logging-config/src/test/java/org/elasticsearch/qa/custom_logging/CustomLoggingConfigIT.java create mode 100644 qa/logging-config/src/test/resources/org/elasticsearch/common/logging/json_layout/log4j2.properties create mode 100644 qa/logging-config/src/test/resources/plugin-security.policy rename qa/unconfigured-node-name/src/test/java/org/elasticsearch/unconfigured_node_name/{NodeNameInLogsIT.java => JsonLogsFormatAndParseIT.java} (92%) create mode 100644 server/src/main/java/org/elasticsearch/common/logging/ESJsonLayout.java create mode 100644 server/src/main/java/org/elasticsearch/common/logging/JsonThrowablePatternConverter.java create mode 100644 server/src/main/java/org/elasticsearch/common/logging/NodeAndClusterIdConverter.java create mode 100644 server/src/main/java/org/elasticsearch/common/logging/NodeAndClusterIdStateListener.java create mode 100644 server/src/test/java/org/elasticsearch/common/logging/JsonThrowablePatternConverterTests.java create mode 100644 test/framework/src/main/java/org/elasticsearch/common/logging/JsonLogLine.java create mode 100644 test/framework/src/main/java/org/elasticsearch/common/logging/JsonLogsIntegTestCase.java create mode 100644 test/framework/src/main/java/org/elasticsearch/common/logging/JsonLogsStream.java delete mode 100644 test/framework/src/main/java/org/elasticsearch/common/logging/NodeNameInLogsIntegTestCase.java diff --git a/distribution/archives/integ-test-zip/build.gradle b/distribution/archives/integ-test-zip/build.gradle index 30fa4d3c03805..d79971907b50d 100644 --- a/distribution/archives/integ-test-zip/build.gradle +++ b/distribution/archives/integ-test-zip/build.gradle @@ -27,7 +27,7 @@ integTestRunner { */ if (System.getProperty("tests.rest.cluster") == null) { systemProperty 'tests.logfile', - "${ -> integTest.nodes[0].homeDir}/logs/${ -> integTest.nodes[0].clusterName }.log" + "${ -> integTest.nodes[0].homeDir}/logs/${ -> integTest.nodes[0].clusterName }_server.json" } else { systemProperty 'tests.logfile', '--external--' } diff --git a/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/NodeNameInLogsIT.java b/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/JsonLogsFormatAndParseIT.java similarity index 88% rename from distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/NodeNameInLogsIT.java rename to distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/JsonLogsFormatAndParseIT.java index a854e6e66462a..12c916946085b 100644 --- a/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/NodeNameInLogsIT.java +++ b/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/JsonLogsFormatAndParseIT.java @@ -19,11 +19,11 @@ package org.elasticsearch.test.rest; -import org.elasticsearch.common.logging.NodeNameInLogsIntegTestCase; +import org.elasticsearch.common.logging.JsonLogsIntegTestCase; import org.hamcrest.Matcher; -import java.io.IOException; import java.io.BufferedReader; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -32,7 +32,7 @@ import static org.hamcrest.Matchers.is; -public class NodeNameInLogsIT extends NodeNameInLogsIntegTestCase { +public class JsonLogsFormatAndParseIT extends JsonLogsIntegTestCase { @Override protected Matcher nodeNameMatcher() { return is("node-0"); @@ -41,7 +41,7 @@ protected Matcher nodeNameMatcher() { @Override protected BufferedReader openReader(Path logFile) { assumeFalse("Skipping test because it is being run against an external cluster.", - logFile.getFileName().toString().equals("--external--")); + logFile.getFileName().toString().equals("--external--")); return AccessController.doPrivileged((PrivilegedAction) () -> { try { return Files.newBufferedReader(logFile, StandardCharsets.UTF_8); diff --git a/distribution/docker/src/docker/config/log4j2.properties b/distribution/docker/src/docker/config/log4j2.properties index 9ad290ad82679..73420a047edc5 100644 --- a/distribution/docker/src/docker/config/log4j2.properties +++ b/distribution/docker/src/docker/config/log4j2.properties @@ -1,9 +1,43 @@ status = error -appender.console.type = Console -appender.console.name = console -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n +# log action execution errors for easier debugging +logger.action.name = org.elasticsearch.action +logger.action.level = debug + +appender.rolling.type = Console +appender.rolling.name = rolling +appender.rolling.layout.type = ESJsonLayout +appender.rolling.layout.type_name = server rootLogger.level = info -rootLogger.appenderRef.console.ref = console +rootLogger.appenderRef.rolling.ref = rolling + +appender.deprecation_rolling.type = Console +appender.deprecation_rolling.name = deprecation_rolling +appender.deprecation_rolling.layout.type = ESJsonLayout +appender.deprecation_rolling.layout.type_name = deprecation + +logger.deprecation.name = org.elasticsearch.deprecation +logger.deprecation.level = warn +logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling +logger.deprecation.additivity = false + +appender.index_search_slowlog_rolling.type = Console +appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling +appender.index_search_slowlog_rolling.layout.type = ESJsonLayout +appender.index_search_slowlog_rolling.layout.type_name = index_search_slowlog + +logger.index_search_slowlog_rolling.name = index.search.slowlog +logger.index_search_slowlog_rolling.level = trace +logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling +logger.index_search_slowlog_rolling.additivity = false + +appender.index_indexing_slowlog_rolling.type = Console +appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling +appender.index_indexing_slowlog_rolling.layout.type = ESJsonLayout +appender.index_indexing_slowlog_rolling.layout.type_name = index_indexing_slowlog + +logger.index_indexing_slowlog.name = index.indexing.slowlog.index +logger.index_indexing_slowlog.level = trace +logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling +logger.index_indexing_slowlog.additivity = false diff --git a/distribution/src/config/log4j2.properties b/distribution/src/config/log4j2.properties index 6de21cd48f67b..45bf720902c1c 100644 --- a/distribution/src/config/log4j2.properties +++ b/distribution/src/config/log4j2.properties @@ -9,12 +9,14 @@ appender.console.name = console appender.console.layout.type = PatternLayout appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n +######## Server JSON ############################ appender.rolling.type = RollingFile appender.rolling.name = rolling -appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log -appender.rolling.layout.type = PatternLayout -appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %.-10000m%n -appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.log.gz +appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_server.json +appender.rolling.layout.type = ESJsonLayout +appender.rolling.layout.type_name = server + +appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.json.gz appender.rolling.policies.type = Policies appender.rolling.policies.time.type = TimeBasedTriggeringPolicy appender.rolling.policies.time.interval = 1 @@ -29,58 +31,144 @@ appender.rolling.strategy.action.condition.type = IfFileName appender.rolling.strategy.action.condition.glob = ${sys:es.logs.cluster_name}-* appender.rolling.strategy.action.condition.nested_condition.type = IfAccumulatedFileSize appender.rolling.strategy.action.condition.nested_condition.exceeds = 2GB +################################################ +######## Server - old style pattern ########### +appender.rolling_old.type = RollingFile +appender.rolling_old.name = rolling_old +appender.rolling_old.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log +appender.rolling_old.layout.type = PatternLayout +appender.rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n + +appender.rolling_old.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.log.gz +appender.rolling_old.policies.type = Policies +appender.rolling_old.policies.time.type = TimeBasedTriggeringPolicy +appender.rolling_old.policies.time.interval = 1 +appender.rolling_old.policies.time.modulate = true +appender.rolling_old.policies.size.type = SizeBasedTriggeringPolicy +appender.rolling_old.policies.size.size = 128MB +appender.rolling_old.strategy.type = DefaultRolloverStrategy +appender.rolling_old.strategy.fileIndex = nomax +appender.rolling_old.strategy.action.type = Delete +appender.rolling_old.strategy.action.basepath = ${sys:es.logs.base_path} +appender.rolling_old.strategy.action.condition.type = IfFileName +appender.rolling_old.strategy.action.condition.glob = ${sys:es.logs.cluster_name}-* +appender.rolling_old.strategy.action.condition.nested_condition.type = IfAccumulatedFileSize +appender.rolling_old.strategy.action.condition.nested_condition.exceeds = 2GB +################################################ rootLogger.level = info rootLogger.appenderRef.console.ref = console rootLogger.appenderRef.rolling.ref = rolling +rootLogger.appenderRef.rolling_old.ref = rolling_old +######## Deprecation JSON ####################### appender.deprecation_rolling.type = RollingFile appender.deprecation_rolling.name = deprecation_rolling -appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log -appender.deprecation_rolling.layout.type = PatternLayout -appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %.-10000m%n -appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.log.gz +appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.json +appender.deprecation_rolling.layout.type = ESJsonLayout +appender.deprecation_rolling.layout.type_name = deprecation + +appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.json.gz appender.deprecation_rolling.policies.type = Policies appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy appender.deprecation_rolling.policies.size.size = 1GB appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy appender.deprecation_rolling.strategy.max = 4 +################################################# +######## Deprecation - old style pattern ####### +appender.deprecation_rolling_old.type = RollingFile +appender.deprecation_rolling_old.name = deprecation_rolling_old +appender.deprecation_rolling_old.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log +appender.deprecation_rolling_old.layout.type = PatternLayout +appender.deprecation_rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n +appender.deprecation_rolling_old.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\ + _deprecation-%i.log.gz +appender.deprecation_rolling_old.policies.type = Policies +appender.deprecation_rolling_old.policies.size.type = SizeBasedTriggeringPolicy +appender.deprecation_rolling_old.policies.size.size = 1GB +appender.deprecation_rolling_old.strategy.type = DefaultRolloverStrategy +appender.deprecation_rolling_old.strategy.max = 4 +################################################# logger.deprecation.name = org.elasticsearch.deprecation logger.deprecation.level = warn logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling +logger.deprecation.appenderRef.deprecation_rolling_old.ref = deprecation_rolling_old logger.deprecation.additivity = false +######## Search slowlog JSON #################### appender.index_search_slowlog_rolling.type = RollingFile appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling -appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog.log -appender.index_search_slowlog_rolling.layout.type = PatternLayout -appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] [%node_name]%marker %.-10000m%n -appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog-%i.log.gz +appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs\ + .cluster_name}_index_search_slowlog.json +appender.index_search_slowlog_rolling.layout.type = ESJsonLayout +appender.index_search_slowlog_rolling.layout.type_name = index_search_slowlog + +appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs\ + .cluster_name}_index_search_slowlog-%i.json.gz appender.index_search_slowlog_rolling.policies.type = Policies appender.index_search_slowlog_rolling.policies.size.type = SizeBasedTriggeringPolicy appender.index_search_slowlog_rolling.policies.size.size = 1GB appender.index_search_slowlog_rolling.strategy.type = DefaultRolloverStrategy appender.index_search_slowlog_rolling.strategy.max = 4 +################################################# +######## Search slowlog - old style pattern #### +appender.index_search_slowlog_rolling_old.type = RollingFile +appender.index_search_slowlog_rolling_old.name = index_search_slowlog_rolling_old +appender.index_search_slowlog_rolling_old.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\ + _index_search_slowlog.log +appender.index_search_slowlog_rolling_old.layout.type = PatternLayout +appender.index_search_slowlog_rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n +appender.index_search_slowlog_rolling_old.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\ + _index_search_slowlog-%i.log.gz +appender.index_search_slowlog_rolling_old.policies.type = Policies +appender.index_search_slowlog_rolling_old.policies.size.type = SizeBasedTriggeringPolicy +appender.index_search_slowlog_rolling_old.policies.size.size = 1GB +appender.index_search_slowlog_rolling_old.strategy.type = DefaultRolloverStrategy +appender.index_search_slowlog_rolling_old.strategy.max = 4 +################################################# logger.index_search_slowlog_rolling.name = index.search.slowlog logger.index_search_slowlog_rolling.level = trace logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling +logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling_old.ref = index_search_slowlog_rolling_old logger.index_search_slowlog_rolling.additivity = false +######## Indexing slowlog JSON ################## appender.index_indexing_slowlog_rolling.type = RollingFile appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling -appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog.log -appender.index_indexing_slowlog_rolling.layout.type = PatternLayout -appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] [%node_name]%marker %.-10000m%n -appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog-%i.log.gz +appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\ + _index_indexing_slowlog.json +appender.index_indexing_slowlog_rolling.layout.type = ESJsonLayout +appender.index_indexing_slowlog_rolling.layout.type_name = index_indexing_slowlog + +appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\ + _index_indexing_slowlog-%i.json.gz appender.index_indexing_slowlog_rolling.policies.type = Policies appender.index_indexing_slowlog_rolling.policies.size.type = SizeBasedTriggeringPolicy appender.index_indexing_slowlog_rolling.policies.size.size = 1GB appender.index_indexing_slowlog_rolling.strategy.type = DefaultRolloverStrategy appender.index_indexing_slowlog_rolling.strategy.max = 4 +################################################# +######## Indexing slowlog - old style pattern ## +appender.index_indexing_slowlog_rolling_old.type = RollingFile +appender.index_indexing_slowlog_rolling_old.name = index_indexing_slowlog_rolling_old +appender.index_indexing_slowlog_rolling_old.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\ + _index_indexing_slowlog.log +appender.index_indexing_slowlog_rolling_old.layout.type = PatternLayout +appender.index_indexing_slowlog_rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n + +appender.index_indexing_slowlog_rolling_old.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\ + _index_indexing_slowlog-%i.log.gz +appender.index_indexing_slowlog_rolling_old.policies.type = Policies +appender.index_indexing_slowlog_rolling_old.policies.size.type = SizeBasedTriggeringPolicy +appender.index_indexing_slowlog_rolling_old.policies.size.size = 1GB +appender.index_indexing_slowlog_rolling_old.strategy.type = DefaultRolloverStrategy +appender.index_indexing_slowlog_rolling_old.strategy.max = 4 +################################################# logger.index_indexing_slowlog.name = index.indexing.slowlog.index logger.index_indexing_slowlog.level = trace logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling +logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling_old.ref = index_indexing_slowlog_rolling_old logger.index_indexing_slowlog.additivity = false diff --git a/docs/reference/migration/migrate_7_0.asciidoc b/docs/reference/migration/migrate_7_0.asciidoc index 9f99604318aa9..313fdfdfafbe5 100644 --- a/docs/reference/migration/migrate_7_0.asciidoc +++ b/docs/reference/migration/migrate_7_0.asciidoc @@ -25,6 +25,7 @@ See also <> and <>. * <> * <> * <> +* <> [float] === Indices created before 7.0 @@ -58,3 +59,4 @@ include::migrate_7_0/scripting.asciidoc[] include::migrate_7_0/snapshotstats.asciidoc[] include::migrate_7_0/restclient.asciidoc[] include::migrate_7_0/low_level_restclient.asciidoc[] +include::migrate_7_0/logging.asciidoc[] diff --git a/docs/reference/migration/migrate_7_0/logging.asciidoc b/docs/reference/migration/migrate_7_0/logging.asciidoc new file mode 100644 index 0000000000000..0385397b31619 --- /dev/null +++ b/docs/reference/migration/migrate_7_0/logging.asciidoc @@ -0,0 +1,33 @@ +[float] +[[breaking_70_logging_changes]] +=== Logging changes + +[float] +==== New JSON format log files in `log` directory + +Elasticsearch now will produce additional log files in JSON format. They will be stored in `*.json` suffix files. +Following files should be expected now in log directory: +* ${cluster_name}_server.json +* ${cluster_name}_deprecation.json +* ${cluster_name}_index_search_slowlog.json +* ${cluster_name}_index_indexing_slowlog.json +* ${cluster_name}.log +* ${cluster_name}_deprecation.log +* ${cluster_name}_index_search_slowlog.log +* ${cluster_name}_index_indexing_slowlog.log +* ${cluster_name}_audit.json +* gc.log + +Note: You can configure which of these files are written by editing `log4j2.properties`. + +[float] +==== Log files ending with `*.log` deprecated +Log files with the `.log` file extension using the old pattern layout format +are now considered deprecated and the newly added JSON log file format with +the `.json` file extension should be used instead. +Note: GC logs which are written to the file `gc.log` will not be changed. + +[float] +==== Docker output in JSON format + +All Docker console logs are now in JSON format. You can distinguish logs streams with the `type` field. diff --git a/docs/reference/setup/logging-config.asciidoc b/docs/reference/setup/logging-config.asciidoc index f477a14bb6d3d..dcea83a7f5d67 100644 --- a/docs/reference/setup/logging-config.asciidoc +++ b/docs/reference/setup/logging-config.asciidoc @@ -20,43 +20,62 @@ will resolve to `/var/log/elasticsearch/production.log`. [source,properties] -------------------------------------------------- +######## Server JSON ############################ appender.rolling.type = RollingFile <1> appender.rolling.name = rolling -appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log <2> -appender.rolling.layout.type = PatternLayout -appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %.-10000m%n -appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.log.gz <3> +appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_server.json <2> +appender.rolling.layout.type = ESJsonLayout <3> +appender.rolling.layout.type_name = server <4> +appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.json.gz <5> appender.rolling.policies.type = Policies -appender.rolling.policies.time.type = TimeBasedTriggeringPolicy <4> -appender.rolling.policies.time.interval = 1 <5> -appender.rolling.policies.time.modulate = true <6> -appender.rolling.policies.size.type = SizeBasedTriggeringPolicy <7> -appender.rolling.policies.size.size = 256MB <8> +appender.rolling.policies.time.type = TimeBasedTriggeringPolicy <6> +appender.rolling.policies.time.interval = 1 <7> +appender.rolling.policies.time.modulate = true <8> +appender.rolling.policies.size.type = SizeBasedTriggeringPolicy <9> +appender.rolling.policies.size.size = 256MB <10> appender.rolling.strategy.type = DefaultRolloverStrategy appender.rolling.strategy.fileIndex = nomax -appender.rolling.strategy.action.type = Delete <9> +appender.rolling.strategy.action.type = Delete <11> appender.rolling.strategy.action.basepath = ${sys:es.logs.base_path} -appender.rolling.strategy.action.condition.type = IfFileName <10> -appender.rolling.strategy.action.condition.glob = ${sys:es.logs.cluster_name}-* <11> -appender.rolling.strategy.action.condition.nested_condition.type = IfAccumulatedFileSize <12> -appender.rolling.strategy.action.condition.nested_condition.exceeds = 2GB <13> +appender.rolling.strategy.action.condition.type = IfFileName <12> +appender.rolling.strategy.action.condition.glob = ${sys:es.logs.cluster_name}-* <13> +appender.rolling.strategy.action.condition.nested_condition.type = IfAccumulatedFileSize <14> +appender.rolling.strategy.action.condition.nested_condition.exceeds = 2GB <15> +################################################ -------------------------------------------------- <1> Configure the `RollingFile` appender -<2> Log to `/var/log/elasticsearch/production.log` -<3> Roll logs to `/var/log/elasticsearch/production-yyyy-MM-dd-i.log`; logs +<2> Log to `/var/log/elasticsearch/production.json` +<3> Use JSON layout. +<4> `type_name` is a flag populating the `type` field in a `ESJsonLayout`. + It can be used to distinguish different types of logs more easily when parsing them. +<5> Roll logs to `/var/log/elasticsearch/production-yyyy-MM-dd-i.json`; logs will be compressed on each roll and `i` will be incremented -<4> Use a time-based roll policy -<5> Roll logs on a daily basis -<6> Align rolls on the day boundary (as opposed to rolling every twenty-four +<6> Use a time-based roll policy +<7> Roll logs on a daily basis +<8> Align rolls on the day boundary (as opposed to rolling every twenty-four hours) -<7> Using a size-based roll policy -<8> Roll logs after 256 MB -<9> Use a delete action when rolling logs -<10> Only delete logs matching a file pattern -<11> The pattern is to only delete the main logs -<12> Only delete if we have accumulated too many compressed logs -<13> The size condition on the compressed logs is 2 GB +<9> Using a size-based roll policy +<10> Roll logs after 256 MB +<11> Use a delete action when rolling logs +<12> Only delete logs matching a file pattern +<13> The pattern is to only delete the main logs +<14> Only delete if we have accumulated too many compressed logs +<15> The size condition on the compressed logs is 2 GB + +[source,properties] +-------------------------------------------------- +######## Server - old style pattern ########### +appender.rolling_old.type = RollingFile +appender.rolling_old.name = rolling_old +appender.rolling_old.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_server.log <1> +appender.rolling_old.layout.type = PatternLayout +appender.rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n +appender.rolling_old.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.old_log.gz + +-------------------------------------------------- +<1> The configuration for `old style` pattern appenders. These logs will be saved in `*.log` files and if archived will be in `* +.log.gz` files. Note that these should be considered deprecated and will be removed in the future. NOTE: Log4j's configuration parsing gets confused by any extraneous whitespace; if you copy and paste any Log4j settings on this page, or enter any Log4j @@ -194,3 +213,38 @@ files (four rolled logs, and the active log). You can disable it in the `config/log4j2.properties` file by setting the deprecation log level to `error`. + + +[float] +[[json-logging]] +=== JSON log format + +To make parsing Elasticsearch logs easier, logs are now printed in a JSON format. +This is configured by a Log4J layout property `appender.rolling.layout.type = ESJsonLayout`. +This layout requires a `type_name` attribute to be set which is used to distinguish +logs streams when parsing. +[source,properties] +-------------------------------------------------- +appender.rolling.layout.type = ESJsonLayout +appender.rolling.layout.type_name = server +-------------------------------------------------- +:es-json-layout-java-doc: {elasticsearch-javadoc}/org/elasticsearch/common/logging/ESJsonLayout.html + +Each line contains a single JSON document with the properties configured in `ESJsonLayout`. +See this class {es-json-layout-java-doc}[javadoc] for more details. +However if a JSON document contains an exception, it will be printed over multiple lines. +The first line will contain regular properties and subsequent lines will contain the +stacktrace formatted as a JSON array. + + +NOTE: You can still use your own custom layout. To do that replace the line +`appender.rolling.layout.type` with a different layout. See sample below: +[source,properties] +-------------------------------------------------- +appender.rolling.type = RollingFile +appender.rolling.name = rolling +appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_server.log +appender.rolling.layout.type = PatternLayout +appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %.-10000m%n +appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.log.gz +-------------------------------------------------- \ No newline at end of file diff --git a/qa/die-with-dignity/build.gradle b/qa/die-with-dignity/build.gradle index 26d567ca3ef6c..3b2e21fd557e7 100644 --- a/qa/die-with-dignity/build.gradle +++ b/qa/die-with-dignity/build.gradle @@ -28,7 +28,7 @@ integTestRunner { systemProperty 'tests.security.manager', 'false' systemProperty 'tests.system_call_filter', 'false' systemProperty 'pidfile', "${-> integTest.getNodes().get(0).pidFile}" - systemProperty 'log', "${-> integTest.getNodes().get(0).homeDir}/logs/${-> integTest.getNodes().get(0).clusterName}.log" + systemProperty 'log', "${-> integTest.getNodes().get(0).homeDir}/logs/${-> integTest.getNodes().get(0).clusterName}_server.json" systemProperty 'runtime.java.home', "${project.runtimeJavaHome}" } diff --git a/qa/die-with-dignity/src/test/java/org/elasticsearch/qa/die_with_dignity/DieWithDignityIT.java b/qa/die-with-dignity/src/test/java/org/elasticsearch/qa/die_with_dignity/DieWithDignityIT.java index 9250122025c0a..16398b380cfe1 100644 --- a/qa/die-with-dignity/src/test/java/org/elasticsearch/qa/die_with_dignity/DieWithDignityIT.java +++ b/qa/die-with-dignity/src/test/java/org/elasticsearch/qa/die_with_dignity/DieWithDignityIT.java @@ -21,10 +21,14 @@ import org.apache.http.ConnectionClosedException; import org.apache.lucene.util.Constants; +import org.elasticsearch.cli.Terminal; import org.elasticsearch.client.Request; import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.common.logging.JsonLogLine; +import org.elasticsearch.common.logging.JsonLogsStream; import org.elasticsearch.test.rest.ESRestTestCase; import org.hamcrest.Matcher; +import org.hamcrest.Matchers; import java.io.BufferedReader; import java.io.IOException; @@ -34,10 +38,12 @@ import java.nio.file.Path; import java.util.Iterator; import java.util.List; +import java.util.stream.Stream; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.either; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.hasToString; import static org.hamcrest.Matchers.instanceOf; @@ -53,7 +59,7 @@ public void testDieWithDignity() throws Exception { final int pid = Integer.parseInt(pidFileLines.get(0)); Files.delete(pidFile); IOException e = expectThrows(IOException.class, - () -> client().performRequest(new Request("GET", "/_die_with_dignity"))); + () -> client().performRequest(new Request("GET", "/_die_with_dignity"))); Matcher failureMatcher = instanceOf(ConnectionClosedException.class); if (Constants.WINDOWS) { /* @@ -64,9 +70,9 @@ public void testDieWithDignity() throws Exception { * https://issues.apache.org/jira/browse/HTTPASYNC-134 * * So we catch it here and consider it "ok". - */ + */ failureMatcher = either(failureMatcher) - .or(hasToString(containsString("An existing connection was forcibly closed by the remote host"))); + .or(hasToString(containsString("An existing connection was forcibly closed by the remote host"))); } assertThat(e, failureMatcher); @@ -85,28 +91,62 @@ public void testDieWithDignity() throws Exception { } }); - // parse the logs and ensure that Elasticsearch died with the expected cause - final List lines = Files.readAllLines(PathUtils.get(System.getProperty("log"))); + try { + // parse the logs and ensure that Elasticsearch died with the expected cause + Path path = PathUtils.get(System.getProperty("log")); + try (Stream stream = JsonLogsStream.from(path)) { + final Iterator it = stream.iterator(); - final Iterator it = lines.iterator(); + boolean fatalError = false; + boolean fatalErrorInThreadExiting = false; - boolean fatalError = false; - boolean fatalErrorInThreadExiting = false; + while (it.hasNext() && (fatalError == false || fatalErrorInThreadExiting == false)) { + final JsonLogLine line = it.next(); + if (isFatalError(line)) { + fatalError = true; + } else if (isFatalErrorInThreadExiting(line) || isWarnExceptionReceived(line)) { + fatalErrorInThreadExiting = true; + assertThat(line.stacktrace(), + hasItem(Matchers.containsString("java.lang.OutOfMemoryError: die with dignity"))); + } + } - while (it.hasNext() && (fatalError == false || fatalErrorInThreadExiting == false)) { - final String line = it.next(); - if (line.matches(".*\\[ERROR\\]\\[o\\.e\\.ExceptionsHelper\\s*\\] \\[node-0\\] fatal error")) { - fatalError = true; - } else if (line.matches(".*\\[ERROR\\]\\[o\\.e\\.b\\.ElasticsearchUncaughtExceptionHandler\\] \\[node-0\\]" - + " fatal error in thread \\[Thread-\\d+\\], exiting$")) { - fatalErrorInThreadExiting = true; - assertTrue(it.hasNext()); - assertThat(it.next(), equalTo("java.lang.OutOfMemoryError: die with dignity")); + assertTrue(fatalError); + assertTrue(fatalErrorInThreadExiting); } + } catch (AssertionError ae) { + Path path = PathUtils.get(System.getProperty("log")); + debugLogs(path); + throw ae; } + } + + private boolean isWarnExceptionReceived(JsonLogLine line) { + return line.level().equals("WARN") + && line.component().equals("o.e.h.AbstractHttpServerTransport") + && line.nodeName().equals("node-0") + && line.message().contains("caught exception while handling client http traffic"); + } + + private void debugLogs(Path path) throws IOException { + try (BufferedReader reader = Files.newBufferedReader(path)) { + Terminal terminal = Terminal.DEFAULT; + reader.lines().forEach(line -> terminal.println(line)); + } + } + + private boolean isFatalErrorInThreadExiting(JsonLogLine line) { + return line.level().equals("ERROR") + && line.component().equals("o.e.b.ElasticsearchUncaughtExceptionHandler") + && line.nodeName().equals("node-0") + && line.message().matches("fatal error in thread \\[Thread-\\d+\\], exiting$"); + } - assertTrue(fatalError); - assertTrue(fatalErrorInThreadExiting); + private boolean isFatalError(JsonLogLine line) { + return line.level().equals("ERROR") + && line.component().equals("o.e.ExceptionsHelper") + && line.nodeName().equals("node-0") + && line.message().contains("fatal error"); } @Override diff --git a/qa/logging-config/build.gradle b/qa/logging-config/build.gradle new file mode 100644 index 0000000000000..0abdc1247514a --- /dev/null +++ b/qa/logging-config/build.gradle @@ -0,0 +1,42 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +apply plugin: 'elasticsearch.standalone-rest-test' +apply plugin: 'elasticsearch.rest-test' +apply plugin: 'elasticsearch.standalone-test' + +integTestCluster { + autoSetInitialMasterNodes = false + autoSetHostsProvider = false + /** + * Provide a custom log4j configuration where layout is an old style pattern and confirm that Elasticsearch + * can successfully startup. + */ + extraConfigFile 'log4j2.properties', 'custom-log4j2.properties' +} + +integTestRunner { + systemProperty 'tests.logfile', + "${ -> integTest.nodes[0].homeDir}/logs/${ -> integTest.nodes[0].clusterName }_server.log" +} + +unitTest { + systemProperty 'tests.security.manager', 'false' +} diff --git a/qa/logging-config/custom-log4j2.properties b/qa/logging-config/custom-log4j2.properties new file mode 100644 index 0000000000000..b225d7cd550cf --- /dev/null +++ b/qa/logging-config/custom-log4j2.properties @@ -0,0 +1,31 @@ + +status = error + +# log action execution errors for easier debugging +logger.action.name = org.elasticsearch.action +logger.action.level = debug + +appender.rolling.type = RollingFile +appender.rolling.name = rolling +appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_server.log +appender.rolling.layout.type = PatternLayout +appender.rolling.layout.pattern =%notEmpty{%node_name} %notEmpty{%node_and_cluster_id} %notEmpty{${sys:es.logs.cluster_name}} %m%n + +appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.log.gz +appender.rolling.policies.type = Policies +appender.rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.rolling.policies.time.interval = 1 +appender.rolling.policies.time.modulate = true +appender.rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.rolling.policies.size.size = 128MB +appender.rolling.strategy.type = DefaultRolloverStrategy +appender.rolling.strategy.fileIndex = nomax +appender.rolling.strategy.action.type = Delete +appender.rolling.strategy.action.basepath = ${sys:es.logs.base_path} +appender.rolling.strategy.action.condition.type = IfFileName +appender.rolling.strategy.action.condition.glob = ${sys:es.logs.cluster_name}-* +appender.rolling.strategy.action.condition.nested_condition.type = IfAccumulatedFileSize +appender.rolling.strategy.action.condition.nested_condition.exceeds = 2GB + +rootLogger.level = info +rootLogger.appenderRef.rolling.ref = rolling diff --git a/qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLoggerTests.java b/qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLoggerTests.java new file mode 100644 index 0000000000000..b484ba90a4da3 --- /dev/null +++ b/qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLoggerTests.java @@ -0,0 +1,232 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.logging; + +import org.apache.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.config.Configurator; +import org.elasticsearch.cli.UserException; +import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.test.ESTestCase; +import org.hamcrest.FeatureMatcher; +import org.hamcrest.Matcher; +import org.hamcrest.Matchers; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * This test confirms JSON log structure is properly formatted and can be parsed. + * It has to be in a org.elasticsearch.common.logging package to use PrefixLogger + */ +public class JsonLoggerTests extends ESTestCase { + + @BeforeClass + public static void initNodeName() { + LogConfigurator.setNodeName("sample-name"); + } + + @Override + public void setUp() throws Exception { + super.setUp(); + LogConfigurator.registerErrorListener(); + setupLogging("json_layout"); + } + + @Override + public void tearDown() throws Exception { + LoggerContext context = (LoggerContext) LogManager.getContext(false); + Configurator.shutdown(context); + super.tearDown(); + } + + @SuppressWarnings("unchecked") + public void testJsonLayout() throws IOException { + final Logger testLogger = LogManager.getLogger("test"); + + testLogger.error("This is an error message"); + testLogger.warn("This is a warning message"); + testLogger.info("This is an info message"); + testLogger.debug("This is a debug message"); + testLogger.trace("This is a trace message"); + final Path path = clusterLogsPath(); + try (Stream stream = JsonLogsStream.from(path)) { + List jsonLogs = collectLines(stream); + + assertThat(jsonLogs, Matchers.contains( + logLine("file", Level.ERROR, "sample-name", "test", "This is an error message"), + logLine("file", Level.WARN, "sample-name", "test", "This is a warning message"), + logLine("file", Level.INFO, "sample-name", "test", "This is an info message"), + logLine("file", Level.DEBUG, "sample-name", "test", "This is a debug message"), + logLine("file", Level.TRACE, "sample-name", "test", "This is a trace message") + )); + } + } + + @SuppressWarnings("unchecked") + public void testPrefixLoggerInJson() throws IOException { + Logger shardIdLogger = Loggers.getLogger("shardIdLogger", ShardId.fromString("[indexName][123]")); + shardIdLogger.info("This is an info message with a shardId"); + + Logger prefixLogger = new PrefixLogger(LogManager.getLogger("prefixLogger"), "PREFIX"); + prefixLogger.info("This is an info message with a prefix"); + + final Path path = clusterLogsPath(); + try (Stream stream = JsonLogsStream.from(path)) { + List jsonLogs = collectLines(stream); + assertThat(jsonLogs, Matchers.contains( + logLine("file", Level.INFO, "sample-name", "shardIdLogger", "[indexName][123] This is an info message with a shardId"), + logLine("file", Level.INFO, "sample-name", "prefixLogger", "PREFIX This is an info message with a prefix") + )); + } + } + + public void testJsonInMessage() throws IOException { + final Logger testLogger = LogManager.getLogger("test"); + String json = "{\n" + + " \"terms\" : {\n" + + " \"user\" : [\n" + + " \"u1\",\n" + + " \"u2\",\n" + + " \"u3\"\n" + + " ],\n" + + " \"boost\" : 1.0\n" + + " }\n" + + "}"; + + testLogger.info(json); + + final Path path = clusterLogsPath(); + try (Stream stream = JsonLogsStream.from(path)) { + List jsonLogs = collectLines(stream); + assertThat(jsonLogs, Matchers.contains( + logLine("file", Level.INFO, "sample-name", "test", json) + )); + } + } + + public void testStacktrace() throws IOException { + final Logger testLogger = LogManager.getLogger("test"); + testLogger.error("error message", new Exception("exception message", new RuntimeException("cause message"))); + + final Path path = clusterLogsPath(); + try (Stream stream = JsonLogsStream.from(path)) { + List jsonLogs = collectLines(stream); + assertThat(jsonLogs, Matchers.contains( + Matchers.allOf( + logLine("file", Level.ERROR, "sample-name", "test", "error message"), + stacktraceWith("java.lang.Exception: exception message"), + stacktraceWith("Caused by: java.lang.RuntimeException: cause message") + ) + )); + } + } + + public void testJsonInStacktraceMessageIsSplitted() throws IOException { + final Logger testLogger = LogManager.getLogger("test"); + + String json = "{\n" + + " \"terms\" : {\n" + + " \"user\" : [\n" + + " \"u1\",\n" + + " \"u2\",\n" + + " \"u3\"\n" + + " ],\n" + + " \"boost\" : 1.0\n" + + " }\n" + + "}"; + testLogger.error("error message " + json, new Exception(json)); + + final Path path = clusterLogsPath(); + try (Stream stream = JsonLogsStream.from(path)) { + List jsonLogs = collectLines(stream); + + assertThat(jsonLogs, Matchers.contains( + Matchers.allOf( + //message field will have a single line with json escaped + logLine("file", Level.ERROR, "sample-name", "test", "error message " + json), + + //stacktrace field will have each json line will in a separate array element + stacktraceWith(("java.lang.Exception: " + json).split("\n")) + ) + )); + } + } + + private List collectLines(Stream stream) { + return stream + .skip(1)//skip the first line from super class + .collect(Collectors.toList()); + } + + private Path clusterLogsPath() { + return PathUtils.get(System.getProperty("es.logs.base_path"), System.getProperty("es.logs.cluster_name") + ".log"); + } + + private void setupLogging(final String config) throws IOException, UserException { + setupLogging(config, Settings.EMPTY); + } + + private void setupLogging(final String config, final Settings settings) throws IOException, UserException { + assertFalse("Environment path.home variable should not be set", Environment.PATH_HOME_SETTING.exists(settings)); + final Path configDir = getDataPath(config); + final Settings mergedSettings = Settings.builder() + .put(settings) + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .build(); + // need to use custom config path so we can use a custom log4j2.properties file for the test + final Environment environment = new Environment(mergedSettings, configDir); + LogConfigurator.configure(environment); + } + + private Matcher logLine(String type, Level level, String nodeName, String component, String message) { + return new FeatureMatcher(Matchers.is(true), "logLine", "logLine") { + + @Override + protected Boolean featureValueOf(JsonLogLine actual) { + return actual.type().equals(type) && + actual.level().equals(level.toString()) && + actual.nodeName().equals(nodeName) && + actual.component().equals(component) && + actual.message().equals(message); + } + }; + } + + private Matcher stacktraceWith(String... lines) { + return new FeatureMatcher>(Matchers.hasItems(lines), + "stacktrace", "stacktrace") { + + @Override + protected List featureValueOf(JsonLogLine actual) { + return actual.stacktrace(); + } + }; + } +} diff --git a/qa/logging-config/src/test/java/org/elasticsearch/qa/custom_logging/CustomLoggingConfigIT.java b/qa/logging-config/src/test/java/org/elasticsearch/qa/custom_logging/CustomLoggingConfigIT.java new file mode 100644 index 0000000000000..407d23de99769 --- /dev/null +++ b/qa/logging-config/src/test/java/org/elasticsearch/qa/custom_logging/CustomLoggingConfigIT.java @@ -0,0 +1,72 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.qa.custom_logging; + +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.test.hamcrest.RegexMatcher; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.hamcrest.Matchers; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.List; + +/** + * This test verifies that Elasticsearch can startup successfully with a custom logging config using variables introduced in + * ESJsonLayout + * The intention is to confirm that users can still run their Elasticsearch instances with previous configurations. + */ +public class CustomLoggingConfigIT extends ESRestTestCase { + private static final String NODE_STARTED = ".*node-0.*cluster.uuid.*node.id.*started.*"; + + public void testSuccessfulStartupWithCustomConfig() throws Exception { + assertBusy(() -> { + List lines = readAllLines(getLogFile()); + assertThat(lines, Matchers.hasItem(RegexMatcher.matches(NODE_STARTED))); + }); + } + + private List readAllLines(Path logFile) { + return AccessController.doPrivileged((PrivilegedAction>) () -> { + try { + return Files.readAllLines(logFile, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + } + + @SuppressForbidden(reason = "PathUtils doesn't have permission to read this file") + private Path getLogFile() { + String logFileString = System.getProperty("tests.logfile"); + if (logFileString == null) { + fail("tests.logfile must be set to run this test. It is automatically " + + "set by gradle. If you must set it yourself then it should be the absolute path to the " + + "log file."); + } + return Paths.get(logFileString); + } +} diff --git a/qa/logging-config/src/test/resources/org/elasticsearch/common/logging/json_layout/log4j2.properties b/qa/logging-config/src/test/resources/org/elasticsearch/common/logging/json_layout/log4j2.properties new file mode 100644 index 0000000000000..4bbd0b038ab8a --- /dev/null +++ b/qa/logging-config/src/test/resources/org/elasticsearch/common/logging/json_layout/log4j2.properties @@ -0,0 +1,21 @@ +appender.console.type = Console +appender.console.name = console +appender.console.layout.type = ESJsonLayout +appender.console.layout.type_name = console + +appender.file.type = File +appender.file.name = file +appender.file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log +appender.file.layout.type = ESJsonLayout +appender.file.layout.type_name = file + + +rootLogger.level = info +rootLogger.appenderRef.console.ref = console +rootLogger.appenderRef.file.ref = file + +logger.test.name = test +logger.test.level = trace +logger.test.appenderRef.console.ref = console +logger.test.appenderRef.file.ref = file +logger.test.additivity = false diff --git a/qa/logging-config/src/test/resources/plugin-security.policy b/qa/logging-config/src/test/resources/plugin-security.policy new file mode 100644 index 0000000000000..d0d865c4ede16 --- /dev/null +++ b/qa/logging-config/src/test/resources/plugin-security.policy @@ -0,0 +1,4 @@ +grant { + // Needed to read the log file + permission java.io.FilePermission "${tests.logfile}", "read"; +}; diff --git a/qa/unconfigured-node-name/build.gradle b/qa/unconfigured-node-name/build.gradle index f8fb696e8ca85..5aba0562e03f6 100644 --- a/qa/unconfigured-node-name/build.gradle +++ b/qa/unconfigured-node-name/build.gradle @@ -30,5 +30,5 @@ integTestCluster { integTestRunner { systemProperty 'tests.logfile', - "${ -> integTest.nodes[0].homeDir}/logs/${ -> integTest.nodes[0].clusterName }.log" + "${ -> integTest.nodes[0].homeDir}/logs/${ -> integTest.nodes[0].clusterName }_server.json" } diff --git a/qa/unconfigured-node-name/src/test/java/org/elasticsearch/unconfigured_node_name/NodeNameInLogsIT.java b/qa/unconfigured-node-name/src/test/java/org/elasticsearch/unconfigured_node_name/JsonLogsFormatAndParseIT.java similarity index 92% rename from qa/unconfigured-node-name/src/test/java/org/elasticsearch/unconfigured_node_name/NodeNameInLogsIT.java rename to qa/unconfigured-node-name/src/test/java/org/elasticsearch/unconfigured_node_name/JsonLogsFormatAndParseIT.java index 44d5bb6c900f5..50cc20b0e5789 100644 --- a/qa/unconfigured-node-name/src/test/java/org/elasticsearch/unconfigured_node_name/NodeNameInLogsIT.java +++ b/qa/unconfigured-node-name/src/test/java/org/elasticsearch/unconfigured_node_name/JsonLogsFormatAndParseIT.java @@ -19,11 +19,11 @@ package org.elasticsearch.unconfigured_node_name; -import org.elasticsearch.common.logging.NodeNameInLogsIntegTestCase; +import org.elasticsearch.common.logging.JsonLogsIntegTestCase; import org.hamcrest.Matcher; -import java.io.IOException; import java.io.BufferedReader; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -32,7 +32,7 @@ import static org.hamcrest.Matchers.not; -public class NodeNameInLogsIT extends NodeNameInLogsIntegTestCase { +public class JsonLogsFormatAndParseIT extends JsonLogsIntegTestCase { @Override protected Matcher nodeNameMatcher() { return not(""); diff --git a/qa/vagrant/src/test/resources/packaging/tests/60_systemd.bats b/qa/vagrant/src/test/resources/packaging/tests/60_systemd.bats index 3cf495939aff9..8baa75f38f5bc 100644 --- a/qa/vagrant/src/test/resources/packaging/tests/60_systemd.bats +++ b/qa/vagrant/src/test/resources/packaging/tests/60_systemd.bats @@ -98,7 +98,7 @@ setup() { systemctl start elasticsearch.service wait_for_elasticsearch_status assert_file_exist "/var/run/elasticsearch/elasticsearch.pid" - assert_file_exist "/var/log/elasticsearch/elasticsearch.log" + assert_file_exist "/var/log/elasticsearch/elasticsearch_server.json" # Converts the epoch back in a human readable format run date --date=@$epoch "+%Y-%m-%d %H:%M:%S" diff --git a/qa/vagrant/src/test/resources/packaging/utils/utils.bash b/qa/vagrant/src/test/resources/packaging/utils/utils.bash index 92363d4d4e348..3f577668bf1ec 100644 --- a/qa/vagrant/src/test/resources/packaging/utils/utils.bash +++ b/qa/vagrant/src/test/resources/packaging/utils/utils.bash @@ -428,7 +428,7 @@ describe_port() { } debug_collect_logs() { - local es_logfile="$ESLOG/elasticsearch.log" + local es_logfile="$ESLOG/elasticsearch_server.json" local system_logfile='/var/log/messages' if [ -e "$es_logfile" ]; then diff --git a/server/src/main/java/org/elasticsearch/common/logging/ESJsonLayout.java b/server/src/main/java/org/elasticsearch/common/logging/ESJsonLayout.java new file mode 100644 index 0000000000000..af7cd81f202e3 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/logging/ESJsonLayout.java @@ -0,0 +1,118 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.logging; + +import org.apache.logging.log4j.core.Layout; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.config.Node; +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; +import org.apache.logging.log4j.core.layout.AbstractStringLayout; +import org.apache.logging.log4j.core.layout.ByteBufferDestination; +import org.apache.logging.log4j.core.layout.PatternLayout; +import org.elasticsearch.common.Strings; + +import java.nio.charset.Charset; +import java.util.Map; + +/** + * Formats log events as strings in a json format. + *

+ * The class is wrapping the {@link PatternLayout} with a pattern to format into json. This gives more flexibility and control over how the + * log messages are formatted in {@link org.apache.logging.log4j.core.layout.JsonLayout} + */ +@Plugin(name = "ESJsonLayout", category = Node.CATEGORY, elementType = Layout.ELEMENT_TYPE, printObject = true) +public class ESJsonLayout extends AbstractStringLayout { + /** + * Fields used in a pattern to format a json log line: + *

    + *
  • type - the type of logs. These represent appenders and help docker distinguish log streams.
    • + *
  • timestamp - ISO8601 with additional timezone ID
    • + *
  • level - INFO, WARN etc
    • + *
  • component - logger name, most of the times class name
    • + *
  • cluster.name - taken from sys:es.logs.cluster_name system property because it is always set
    • + *
  • node.name - taken from NodeNamePatternConverter, as it can be set in runtime as hostname when not set in elasticsearch.yml
    • + *
  • node_and_cluster_id - in json as node.id and cluster.uuid - taken from NodeAndClusterIdConverter and present + * once clusterStateUpdate is first received
    • + *
  • message - a json escaped message. Multiline messages will be converted to single line with new line explicitly + * replaced to \n
    • + *
  • exceptionAsJson - in json as a stacktrace field. Only present when throwable is passed as a parameter when using a logger. + * Taken from JsonThrowablePatternConverter
    • + *
+ */ + private static final String PATTERN = "{" + + "\"type\": \"${TYPE}\", " + + "\"timestamp\": \"%d{yyyy-MM-dd'T'HH:mm:ss,SSSZ}\", " + + "\"level\": \"%p\", " + + "\"component\": \"%c{1.}\", " + + "\"cluster.name\": \"${sys:es.logs.cluster_name}\", " + + "\"node.name\": \"%node_name\", " + + "%notEmpty{%node_and_cluster_id, } " + + "\"message\": \"%notEmpty{%enc{%marker}{JSON} }%enc{%.-10000m}{JSON}\" " + + "%exceptionAsJson " + + "}%n"; + + private final PatternLayout patternLayout; + + protected ESJsonLayout(String typeName, Charset charset) { + super(charset); + this.patternLayout = PatternLayout.newBuilder() + .withPattern(pattern(typeName)) + .withAlwaysWriteExceptions(false) + .build(); + } + + private String pattern(String type) { + if (Strings.isEmpty(type)) { + throw new IllegalArgumentException("layout parameter 'type_name' cannot be empty"); + } + return PATTERN.replace("${TYPE}", type); + } + + @PluginFactory + public static ESJsonLayout createLayout(@PluginAttribute("type_name") String type, + @PluginAttribute(value = "charset", defaultString = "UTF-8") Charset charset) { + return new ESJsonLayout(type, charset); + } + + @Override + public String toSerializable(final LogEvent event) { + return patternLayout.toSerializable(event); + } + + @Override + public Map getContentFormat() { + return patternLayout.getContentFormat(); + } + + @Override + public void encode(final LogEvent event, final ByteBufferDestination destination) { + patternLayout.encode(event, destination); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("ESJsonLayout{"); + sb.append("patternLayout=").append(patternLayout); + sb.append('}'); + return sb.toString(); + } +} diff --git a/server/src/main/java/org/elasticsearch/common/logging/JsonThrowablePatternConverter.java b/server/src/main/java/org/elasticsearch/common/logging/JsonThrowablePatternConverter.java new file mode 100644 index 0000000000000..97e712512317b --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/logging/JsonThrowablePatternConverter.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache license, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the license for the specific language governing permissions and + * limitations under the license. + */ +package org.elasticsearch.common.logging; + +import com.fasterxml.jackson.core.io.JsonStringEncoder; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.pattern.ConverterKeys; +import org.apache.logging.log4j.core.pattern.ExtendedThrowablePatternConverter; +import org.apache.logging.log4j.core.pattern.PatternConverter; +import org.apache.logging.log4j.core.pattern.ThrowablePatternConverter; +import org.apache.logging.log4j.util.Strings; + +import java.nio.charset.Charset; +import java.util.StringJoiner; + +/** + + * Outputs the Throwable portion of the LoggingEvent as a Json formatted field with array + * "exception": [ "stacktrace", "lines", "as", "array", "elements" ] + * + * Reusing @link org.apache.logging.log4j.core.pattern.ExtendedThrowablePatternConverter which already converts a Throwable from + * LoggingEvent into a multiline string + */ +@Plugin(name = "JsonThrowablePatternConverter", category = PatternConverter.CATEGORY) +@ConverterKeys({"exceptionAsJson"}) +public final class JsonThrowablePatternConverter extends ThrowablePatternConverter { + private final ExtendedThrowablePatternConverter throwablePatternConverter; + + /** + * Private as only expected to be used by log4j2 newInstance method + */ + private JsonThrowablePatternConverter(final Configuration config, final String[] options) { + super("JsonThrowablePatternConverter", "throwable", options, config); + this.throwablePatternConverter = ExtendedThrowablePatternConverter.newInstance(config, options); + } + + /** + * Gets an instance of the class. + * + * @param config The current Configuration. + * @param options pattern options, may be null. If first element is "short", + * only the first line of the throwable will be formatted. + * @return instance of class. + */ + public static JsonThrowablePatternConverter newInstance(final Configuration config, final String[] options) { + return new JsonThrowablePatternConverter(config, options); + } + + /** + * {@inheritDoc} + */ + @Override + public void format(final LogEvent event, final StringBuilder toAppendTo) { + String consoleStacktrace = formatStacktrace(event); + if (Strings.isNotEmpty(consoleStacktrace)) { + String jsonStacktrace = formatJson(consoleStacktrace); + + toAppendTo.append(", "); + toAppendTo.append(jsonStacktrace); + } + } + + private String formatStacktrace(LogEvent event) { + StringBuilder stringBuilder = new StringBuilder(); + throwablePatternConverter.format(event, stringBuilder); + return stringBuilder.toString(); + } + + private String formatJson(String consoleStacktrace) { + String lineSeparator = options.getSeparator() + "\t|" + options.getSeparator(); + String[] split = consoleStacktrace.split(lineSeparator); + + StringJoiner stringJoiner = new StringJoiner(",\n", "\n\"stacktrace\": [", "]"); + for (String line : split) { + stringJoiner.add(wrapAsJson(line)); + } + return stringJoiner.toString(); + } + + private String wrapAsJson(String line) { + byte[] bytes = JsonStringEncoder.getInstance().quoteAsUTF8(line); + return "\"" + new String(bytes, Charset.defaultCharset()) + "\""; + } + + @Override + public boolean handlesThrowable() { + return true; + } +} diff --git a/server/src/main/java/org/elasticsearch/common/logging/NodeAndClusterIdConverter.java b/server/src/main/java/org/elasticsearch/common/logging/NodeAndClusterIdConverter.java new file mode 100644 index 0000000000000..27437947870b4 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/logging/NodeAndClusterIdConverter.java @@ -0,0 +1,78 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.logging; + +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.pattern.ConverterKeys; +import org.apache.logging.log4j.core.pattern.LogEventPatternConverter; +import org.apache.logging.log4j.core.pattern.PatternConverter; +import org.apache.lucene.util.SetOnce; + +import java.util.Locale; + +/** + * Pattern converter to format the node_and_cluster_id variable into JSON fields node.id and cluster.uuid. + * Keeping those two fields together assures that they will be atomically set and become visible in logs at the same time. + */ +@Plugin(category = PatternConverter.CATEGORY, name = "NodeAndClusterIdConverter") +@ConverterKeys({"node_and_cluster_id"}) +public final class NodeAndClusterIdConverter extends LogEventPatternConverter { + private static final SetOnce nodeAndClusterId = new SetOnce<>(); + + /** + * Called by log4j2 to initialize this converter. + */ + public static NodeAndClusterIdConverter newInstance(@SuppressWarnings("unused") final String[] options) { + return new NodeAndClusterIdConverter(); + } + + public NodeAndClusterIdConverter() { + super("NodeAndClusterId", "node_and_cluster_id"); + } + + /** + * Updates only once the clusterID and nodeId. + * Subsequent executions will throw {@link org.apache.lucene.util.SetOnce.AlreadySetException}. + * + * @param nodeId a nodeId received from cluster state update + * @param clusterUUID a clusterId received from cluster state update + */ + public static void setNodeIdAndClusterId(String nodeId, String clusterUUID) { + nodeAndClusterId.set(formatIds(clusterUUID, nodeId)); + } + + /** + * Formats the node.id and cluster.uuid into json fields. + * + * @param event - a log event is ignored in this method as it uses the nodeId and clusterId to format + */ + @Override + public void format(LogEvent event, StringBuilder toAppendTo) { + if (nodeAndClusterId.get() != null) { + toAppendTo.append(nodeAndClusterId.get()); + } + // nodeId/clusterUuid not received yet, not appending + } + + private static String formatIds(String clusterUUID, String nodeId) { + return String.format(Locale.ROOT, "\"cluster.uuid\": \"%s\", \"node.id\": \"%s\"", clusterUUID, nodeId); + } +} diff --git a/server/src/main/java/org/elasticsearch/common/logging/NodeAndClusterIdStateListener.java b/server/src/main/java/org/elasticsearch/common/logging/NodeAndClusterIdStateListener.java new file mode 100644 index 0000000000000..e8f636238447a --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/logging/NodeAndClusterIdStateListener.java @@ -0,0 +1,77 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.logging; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateObserver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.ThreadContext; + +/** + * The {@link NodeAndClusterIdStateListener} listens to cluster state changes and ONLY when receives the first update + * it sets the clusterUUID and nodeID in log4j pattern converter {@link NodeAndClusterIdConverter}. + * Once the first update is received, it will automatically be de-registered from subsequent updates. + */ +public class NodeAndClusterIdStateListener implements ClusterStateObserver.Listener { + private static final Logger logger = LogManager.getLogger(NodeAndClusterIdStateListener.class); + + private NodeAndClusterIdStateListener() {} + + /** + * Subscribes for the first cluster state update where nodeId and clusterId is present + * and sets these values in {@link NodeAndClusterIdConverter}. + */ + public static void getAndSetNodeIdAndClusterId(ClusterService clusterService, ThreadContext threadContext) { + ClusterState clusterState = clusterService.state(); + ClusterStateObserver observer = new ClusterStateObserver(clusterState, clusterService, null, logger, threadContext); + + observer.waitForNextChange(new NodeAndClusterIdStateListener(), NodeAndClusterIdStateListener::isNodeAndClusterIdPresent); + } + + private static boolean isNodeAndClusterIdPresent(ClusterState clusterState) { + return getNodeId(clusterState) != null && getClusterUUID(clusterState) != null; + } + + private static String getClusterUUID(ClusterState state) { + return state.getMetaData().clusterUUID(); + } + + private static String getNodeId(ClusterState state) { + return state.getNodes().getLocalNodeId(); + } + + @Override + public void onNewClusterState(ClusterState state) { + String nodeId = getNodeId(state); + String clusterUUID = getClusterUUID(state); + + logger.debug("Received cluster state update. Setting nodeId=[{}] and clusterUuid=[{}]", nodeId, clusterUUID); + NodeAndClusterIdConverter.setNodeIdAndClusterId(nodeId, clusterUUID); + } + + @Override + public void onClusterServiceClose() {} + + @Override + public void onTimeout(TimeValue timeout) {} +} diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index dfc538c23260f..0ea5f1e78cf8b 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -67,6 +67,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.logging.NodeAndClusterIdStateListener; import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.network.NetworkService; @@ -683,10 +684,15 @@ public Node start() throws NodeValidationException { transportService.acceptIncomingRequests(); discovery.startInitialJoin(); final TimeValue initialStateTimeout = DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.get(settings); + NodeAndClusterIdStateListener.getAndSetNodeIdAndClusterId(clusterService, + injector.getInstance(ThreadPool.class).getThreadContext()); + if (initialStateTimeout.millis() > 0) { final ThreadPool thread = injector.getInstance(ThreadPool.class); ClusterState clusterState = clusterService.state(); - ClusterStateObserver observer = new ClusterStateObserver(clusterState, clusterService, null, logger, thread.getThreadContext()); + ClusterStateObserver observer = + new ClusterStateObserver(clusterState, clusterService, null, logger, thread.getThreadContext()); + if (clusterState.nodes().getMasterNodeId() == null) { logger.debug("waiting to join the cluster. timeout [{}]", initialStateTimeout); final CountDownLatch latch = new CountDownLatch(1); diff --git a/server/src/test/java/org/elasticsearch/common/logging/JsonThrowablePatternConverterTests.java b/server/src/test/java/org/elasticsearch/common/logging/JsonThrowablePatternConverterTests.java new file mode 100644 index 0000000000000..d72b598f02865 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/logging/JsonThrowablePatternConverterTests.java @@ -0,0 +1,93 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.logging; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.impl.Log4jLogEvent; +import org.apache.logging.log4j.message.SimpleMessage; +import org.elasticsearch.test.ESTestCase; +import org.hamcrest.Matchers; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.StringReader; + +import static org.hamcrest.Matchers.equalTo; + +public class JsonThrowablePatternConverterTests extends ESTestCase { + JsonThrowablePatternConverter converter = JsonThrowablePatternConverter.newInstance(null, null); + + public void testNoStacktrace() throws IOException { + LogEvent event = Log4jLogEvent.newBuilder() + .build(); + String result = format(event); + + JsonLogLine jsonLogLine = JsonLogsStream.from(new BufferedReader(new StringReader(result))) + .findFirst() + .orElseThrow(() -> new AssertionError("no logs parsed")); + + assertThat(jsonLogLine.stacktrace(), Matchers.nullValue()); + } + + public void testStacktraceWithJson() throws IOException { + LogManager.getLogger().info("asdf"); + + String json = "{\n" + + " \"terms\" : {\n" + + " \"user\" : [\n" + + " \"u1\",\n" + + " \"u2\",\n" + + " \"u3\"\n" + + " ],\n" + + " \"boost\" : 1.0\n" + + " }\n" + + "}"; + Exception thrown = new Exception(json); + LogEvent event = Log4jLogEvent.newBuilder() + .setMessage(new SimpleMessage("message")) + .setThrown(thrown) + .build(); + + String result = format(event); + + //confirms exception is correctly parsed + + JsonLogLine jsonLogLine = JsonLogsStream.from(new BufferedReader(new StringReader(result))) + .findFirst() + .orElseThrow(() -> new AssertionError("no logs parsed")); + + int jsonLength = json.split("\n").length; + int stacktraceLength = thrown.getStackTrace().length; + assertThat("stacktrace should formatted in multiple lines", + jsonLogLine.stacktrace().size(), equalTo(jsonLength + stacktraceLength)); + } + + private String format(LogEvent event) { + StringBuilder builder = new StringBuilder(); + converter.format(event, builder); + String jsonStacktraceElement = builder.toString(); + + return "{\"type\": \"console\", \"timestamp\": \"2019-01-03T16:30:53,058+0100\", \"level\": \"DEBUG\", " + + "\"component\": \"o.e.a.s.TransportSearchAction\", \"cluster.name\": \"clustername\", \"node.name\": \"node-0\", " + + "\"cluster.uuid\": \"OG5MkvOrR9azuClJhWvy6Q\", \"node.id\": \"VTShUqmcQG6SzeKY5nn7qA\", \"message\": \"msg msg\" " + + jsonStacktraceElement + "}"; + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/common/logging/JsonLogLine.java b/test/framework/src/main/java/org/elasticsearch/common/logging/JsonLogLine.java new file mode 100644 index 0000000000000..fa8f3d7d27018 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/common/logging/JsonLogLine.java @@ -0,0 +1,158 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.logging; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ObjectParser; + +import java.util.List; + + +/** + * Represents a single log line in a json format. + * Parsing log lines with this class confirms the json format of logs + */ +public class JsonLogLine { + public static final ObjectParser PARSER = createParser(false); + + private String type; + private String timestamp; + private String level; + private String component; + private String clusterName; + private String nodeName; + private String clusterUuid; + private String nodeId; + private String message; + private List stacktrace; + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("JsonLogLine{"); + sb.append("type='").append(type).append('\''); + sb.append(", timestamp='").append(timestamp).append('\''); + sb.append(", level='").append(level).append('\''); + sb.append(", component='").append(component).append('\''); + sb.append(", clusterName='").append(clusterName).append('\''); + sb.append(", nodeName='").append(nodeName).append('\''); + sb.append(", clusterUuid='").append(clusterUuid).append('\''); + sb.append(", nodeId='").append(nodeId).append('\''); + sb.append(", message='").append(message).append('\''); + sb.append(", stacktrace=").append(stacktrace); + sb.append('}'); + return sb.toString(); + } + + public String type() { + return type; + } + + public String timestamp() { + return timestamp; + } + + public String level() { + return level; + } + + public String component() { + return component; + } + + public String clusterName() { + return clusterName; + } + + public String nodeName() { + return nodeName; + } + + public String clusterUuid() { + return clusterUuid; + } + + public String nodeId() { + return nodeId; + } + + public String message() { + return message; + } + + public List stacktrace() { + return stacktrace; + } + + public void setType(String type) { + this.type = type; + } + + public void setTimestamp(String timestamp) { + this.timestamp = timestamp; + } + + public void setLevel(String level) { + this.level = level; + } + + public void setComponent(String component) { + this.component = component; + } + + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + + public void setNodeName(String nodeName) { + this.nodeName = nodeName; + } + + public void setClusterUuid(String clusterUuid) { + this.clusterUuid = clusterUuid; + } + + public void setNodeId(String nodeId) { + this.nodeId = nodeId; + } + + public void setMessage(String message) { + this.message = message; + } + + public void setStacktrace(List stacktrace) { + this.stacktrace = stacktrace; + } + + private static ObjectParser createParser(boolean ignoreUnknownFields) { + ObjectParser parser = new ObjectParser<>("search_template", ignoreUnknownFields, JsonLogLine::new); + parser.declareString(JsonLogLine::setType, new ParseField("type")); + parser.declareString(JsonLogLine::setTimestamp, new ParseField("timestamp")); + parser.declareString(JsonLogLine::setLevel, new ParseField("level")); + parser.declareString(JsonLogLine::setComponent, new ParseField("component")); + parser.declareString(JsonLogLine::setClusterName, new ParseField("cluster.name")); + parser.declareString(JsonLogLine::setNodeName, new ParseField("node.name")); + parser.declareString(JsonLogLine::setClusterUuid, new ParseField("cluster.uuid")); + parser.declareString(JsonLogLine::setNodeId, new ParseField("node.id")); + parser.declareString(JsonLogLine::setMessage, new ParseField("message")); + parser.declareStringArray(JsonLogLine::setStacktrace, new ParseField("stacktrace")); + + return parser; + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/common/logging/JsonLogsIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/common/logging/JsonLogsIntegTestCase.java new file mode 100644 index 0000000000000..d9ba80d6b35de --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/common/logging/JsonLogsIntegTestCase.java @@ -0,0 +1,129 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.logging; + +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.test.rest.ESRestTestCase; + +import java.io.BufferedReader; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Iterator; +import java.util.stream.Stream; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.isEmptyOrNullString; +import static org.hamcrest.Matchers.not; + +/** + * Tests that extend this class verify that all json layout fields appear in the first few log lines after startup + * Fields available upon process startup: type, timestamp, level, component, + * message, node.name, cluster.name. + * Whereas node.id and cluster.uuid are available later once the first clusterState has been received. + * + * + * node.name, cluster.name, node.id, cluster.uuid + * should not change across all log lines + * + * Note that this won't pass for nodes in clusters that don't have the node name defined in elasticsearch.yml and start + * with DEBUG or TRACE level logging. Those nodes log a few lines before the node.name is set by LogConfigurator.setNodeName. + */ +public abstract class JsonLogsIntegTestCase extends ESRestTestCase { + /** + * Number of lines in the log file to check for the node.name, node.id or cluster.uuid. We don't + * just check the entire log file because it could be quite long + */ + private static final int LINES_TO_CHECK = 10; + + /** + * The node name to expect in the log file. + */ + protected abstract org.hamcrest.Matcher nodeNameMatcher(); + + /** + * Open the log file. This is delegated to subclasses because the test + * framework doesn't have permission to read from the log file but + * subclasses can grant themselves that permission. + */ + protected abstract BufferedReader openReader(Path logFile); + + public void testElementsPresentOnAllLinesOfLog() throws IOException { + JsonLogLine firstLine = findFirstLine(); + assertNotNull(firstLine); + + try (Stream stream = JsonLogsStream.from(openReader(getLogFile()))) { + stream.limit(LINES_TO_CHECK) + .forEach(jsonLogLine -> { + assertThat(jsonLogLine.type(), not(isEmptyOrNullString())); + assertThat(jsonLogLine.timestamp(), not(isEmptyOrNullString())); + assertThat(jsonLogLine.level(), not(isEmptyOrNullString())); + assertThat(jsonLogLine.component(), not(isEmptyOrNullString())); + assertThat(jsonLogLine.message(), not(isEmptyOrNullString())); + + // all lines should have the same nodeName and clusterName + assertThat(jsonLogLine.nodeName(), nodeNameMatcher()); + assertThat(jsonLogLine.clusterName(), equalTo(firstLine.clusterName())); + }); + } + } + + private JsonLogLine findFirstLine() throws IOException { + try (Stream stream = JsonLogsStream.from(openReader(getLogFile()))) { + return stream.findFirst() + .orElseThrow(() -> new AssertionError("no logs at all?!")); + } + } + + public void testNodeIdAndClusterIdConsistentOnceAvailable() throws IOException { + try (Stream stream = JsonLogsStream.from(openReader(getLogFile()))) { + Iterator iterator = stream.iterator(); + + JsonLogLine firstLine = null; + while (iterator.hasNext()) { + JsonLogLine jsonLogLine = iterator.next(); + if (jsonLogLine.nodeId() != null) { + firstLine = jsonLogLine; + } + } + assertNotNull(firstLine); + + //once the nodeId and clusterId are received, they should be the same on remaining lines + + int i = 0; + while (iterator.hasNext() && i++ < LINES_TO_CHECK) { + JsonLogLine jsonLogLine = iterator.next(); + assertThat(jsonLogLine.nodeId(), equalTo(firstLine.nodeId())); + assertThat(jsonLogLine.clusterUuid(), equalTo(firstLine.clusterUuid())); + } + } + } + + @SuppressForbidden(reason = "PathUtils doesn't have permission to read this file") + private Path getLogFile() { + String logFileString = System.getProperty("tests.logfile"); + if (logFileString == null) { + fail("tests.logfile must be set to run this test. It is automatically " + + "set by gradle. If you must set it yourself then it should be the absolute path to the " + + "log file."); + } + return Paths.get(logFileString); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/common/logging/JsonLogsStream.java b/test/framework/src/main/java/org/elasticsearch/common/logging/JsonLogsStream.java new file mode 100644 index 0000000000000..28ad649f55a79 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/common/logging/JsonLogsStream.java @@ -0,0 +1,97 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.logging; + +import org.elasticsearch.common.xcontent.DeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Iterator; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +/** + * Returns a stream of json log lines. + * This is intended to be used for easy and readable assertions for logger tests + */ +public class JsonLogsStream { + private final XContentParser parser; + private final BufferedReader reader; + + private JsonLogsStream(BufferedReader reader) throws IOException { + this.reader = reader; + this.parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + reader); + } + + public static Stream from(BufferedReader reader) throws IOException { + return new JsonLogsStream(reader).stream(); + } + + public static Stream from(Path path) throws IOException { + return from(Files.newBufferedReader(path)); + } + + private Stream stream() { + Spliterator spliterator = Spliterators.spliteratorUnknownSize(new JsonIterator(), Spliterator.ORDERED); + return StreamSupport.stream(spliterator, false) + .onClose(this::close); + } + + private void close() { + try { + parser.close(); + reader.close(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private class JsonIterator implements Iterator { + + @Override + public boolean hasNext() { + return parser.isClosed() == false; + } + + @Override + public JsonLogLine next() { + JsonLogLine apply = JsonLogLine.PARSER.apply(parser, null); + nextToken(); + return apply; + } + + private void nextToken() { + try { + parser.nextToken(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/common/logging/NodeNameInLogsIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/common/logging/NodeNameInLogsIntegTestCase.java deleted file mode 100644 index a8a142096e3dd..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/common/logging/NodeNameInLogsIntegTestCase.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.logging; - -import org.elasticsearch.common.SuppressForbidden; -import org.elasticsearch.test.rest.ESRestTestCase; - -import java.io.BufferedReader; -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.regex.Pattern; -import java.util.regex.Matcher; - -import static org.hamcrest.Matchers.containsString; - -/** - * Tests that extend this class verify that the node name appears in the first - * few log lines on startup. Note that this won't pass for clusters that don't - * the node name defined in elasticsearch.yml and start with - * DEBUG or TRACE level logging. Those nodes log a few lines before they - * resolve the node name. - */ -public abstract class NodeNameInLogsIntegTestCase extends ESRestTestCase { - /** - * Number of lines in the log file to check for the node name. We don't - * just check the entire log file because it could be quite long and - * exceptions don't include the node name. - */ - private static final int LINES_TO_CHECK = 10; - - /** - * The node name to expect in the logs file. - */ - protected abstract org.hamcrest.Matcher nodeNameMatcher(); - - /** - * Open the log file. This is delegated to subclasses because the test - * framework doesn't have permission to read from the log file but - * subclasses can grant themselves that permission. - */ - protected abstract BufferedReader openReader(Path logFile); - - public void testNodeNameIsOnAllLinesOfLog() throws IOException { - BufferedReader logReader = openReader(getLogFile()); - try { - String line = logReader.readLine(); - assertNotNull("no logs at all?!", line); - Matcher m = Pattern.compile("\\] \\[([^\\]]+)\\] ").matcher(line); - if (false == m.find()) { - fail("Didn't see the node name in [" + line + "]"); - } - String nodeName = m.group(1); - - assertThat(nodeName, nodeNameMatcher()); - - int lineNumber = 1; - while (true) { - if (lineNumber < LINES_TO_CHECK) { - break; - } - line = logReader.readLine(); - if (line == null) { - break; // eof - } - lineNumber++; - assertThat(line, containsString("] [" + nodeName + "] ")); - } - } finally { - logReader.close(); - } - } - - @SuppressForbidden(reason = "PathUtils doesn't have permission to read this file") - private Path getLogFile() { - String logFileString = System.getProperty("tests.logfile"); - if (null == logFileString) { - fail("tests.logfile must be set to run this test. It is automatically " - + "set by gradle. If you must set it yourself then it should be the absolute path to the " - + "log file."); - } - return Paths.get(logFileString); - } -} diff --git a/x-pack/plugin/ccr/qa/downgrade-to-basic-license/build.gradle b/x-pack/plugin/ccr/qa/downgrade-to-basic-license/build.gradle index 9147d5251b5be..bba9709087a56 100644 --- a/x-pack/plugin/ccr/qa/downgrade-to-basic-license/build.gradle +++ b/x-pack/plugin/ccr/qa/downgrade-to-basic-license/build.gradle @@ -32,7 +32,7 @@ task writeJavaPolicy { javaPolicy.write( [ "grant {", - " permission java.io.FilePermission \"${-> followClusterTest.getNodes().get(0).homeDir}/logs/${-> followClusterTest.getNodes().get(0).clusterName}.log\", \"read\";", + " permission java.io.FilePermission \"${-> followClusterTest.getNodes().get(0).homeDir}/logs/${-> followClusterTest.getNodes().get(0).clusterName}_server.json\", \"read\";", "};" ].join("\n")) } @@ -54,7 +54,8 @@ followClusterTestRunner { systemProperty 'java.security.policy', "file://${buildDir}/tmp/java.policy" systemProperty 'tests.target_cluster', 'follow' systemProperty 'tests.leader_host', "${-> leaderClusterTest.nodes.get(0).httpUri()}" - systemProperty 'log', "${-> followClusterTest.getNodes().get(0).homeDir}/logs/${-> followClusterTest.getNodes().get(0).clusterName}.log" + systemProperty 'log', "${-> followClusterTest.getNodes().get(0).homeDir}/logs/" + + "${-> followClusterTest.getNodes().get(0).clusterName}_server.json" finalizedBy 'leaderClusterTestCluster#stop' } diff --git a/x-pack/plugin/ccr/qa/downgrade-to-basic-license/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java b/x-pack/plugin/ccr/qa/downgrade-to-basic-license/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java index b612d8822437e..8fb305ba06ee6 100644 --- a/x-pack/plugin/ccr/qa/downgrade-to-basic-license/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java +++ b/x-pack/plugin/ccr/qa/downgrade-to-basic-license/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java @@ -11,13 +11,17 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.common.logging.JsonLogLine; +import org.elasticsearch.common.logging.JsonLogsStream; import org.elasticsearch.common.settings.Settings; +import org.hamcrest.FeatureMatcher; +import org.hamcrest.Matcher; +import org.hamcrest.Matchers; import java.io.IOException; -import java.nio.file.Files; -import java.util.Iterator; -import java.util.List; +import java.nio.file.Path; import java.util.Map; +import java.util.stream.Stream; import static org.elasticsearch.common.xcontent.ObjectPath.eval; import static org.hamcrest.Matchers.containsString; @@ -80,25 +84,10 @@ public void testDowngradeRemoteClusterToBasic() throws Exception { // (does not work on windows...) if (Constants.WINDOWS == false) { assertBusy(() -> { - final List lines = Files.readAllLines(PathUtils.get(System.getProperty("log"))); - final Iterator it = lines.iterator(); - boolean warn = false; - while (it.hasNext()) { - final String line = it.next(); - if (line.matches(".*\\[WARN\\s*\\]\\[o\\.e\\.x\\.c\\.a\\.AutoFollowCoordinator\\s*\\] \\[node-0\\] " + - "failure occurred while fetching cluster state for auto follow pattern \\[test_pattern\\]")) { - warn = true; - break; - } + Path path = PathUtils.get(System.getProperty("log")); + try (Stream stream = JsonLogsStream.from(path)) { + assertTrue(stream.anyMatch(autoFollowCoordinatorWarn()::matches)); } - assertTrue(warn); - assertTrue(it.hasNext()); - final String lineAfterWarn = it.next(); - assertThat( - lineAfterWarn, - equalTo("org.elasticsearch.ElasticsearchStatusException: " + - "can not fetch remote cluster state as the remote cluster [leader_cluster] is not licensed for [ccr]; " + - "the license mode [BASIC] on cluster [leader_cluster] does not enable [ccr]")); }); } }); @@ -108,10 +97,26 @@ public void testDowngradeRemoteClusterToBasic() throws Exception { assertThat(e.getMessage(), containsString("the license mode [BASIC] on cluster [leader_cluster] does not enable [ccr]")); } + private Matcher autoFollowCoordinatorWarn() { + return new FeatureMatcher(Matchers.is(true), "autoFollowCoordinatorWarn", "autoFollowCoordinatorWarn") { + + @Override + protected Boolean featureValueOf(JsonLogLine actual) { + return actual.level().equals("WARN") && + actual.component().equals("o.e.x.c.a.AutoFollowCoordinator") && + actual.nodeName().equals("node-0") && + actual.message().contains("failure occurred while fetching cluster state for auto follow pattern [test_pattern]") && + actual.stacktrace().contains("org.elasticsearch.ElasticsearchStatusException: can not fetch remote cluster state " + + "as the remote cluster [leader_cluster] is not licensed for [ccr]; the license mode [BASIC]" + + " on cluster [leader_cluster] does not enable [ccr]"); + } + }; + } + private void createNewIndexAndIndexDocs(RestClient client, String index) throws IOException { Settings settings = Settings.builder() - .put("index.soft_deletes.enabled", true) - .build(); + .put("index.soft_deletes.enabled", true) + .build(); Request request = new Request("PUT", "/" + index); request.setJsonEntity("{\"settings\": " + Strings.toString(settings) + ", \"mappings\": {\"properties\": {\"field\": {\"type\": \"keyword\"}}}}"); From ebe9c9523080133d547f04d9d67c78c8d6f95fbd Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Tue, 29 Jan 2019 10:23:50 +0200 Subject: [PATCH 002/100] [ML] Audit all errors during job deletion (#37933) This commit moves the auditing of job deletion related errors to the final listener in the job delete action. This ensures any error that occurs during job deletion is audited. --- .../xpack/ml/action/TransportDeleteJobAction.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java index 90d8c6e677a83..cc3b704d77252 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java @@ -181,7 +181,10 @@ protected void masterOperation(Task task, DeleteJobAction.Request request, Clust // The listener that will be executed at the end of the chain will notify all listeners ActionListener finalListener = ActionListener.wrap( ack -> notifyListeners(request.getJobId(), ack, null), - e -> notifyListeners(request.getJobId(), null, e) + e -> { + notifyListeners(request.getJobId(), null, e); + auditor.error(request.getJobId(), Messages.getMessage(Messages.JOB_AUDIT_DELETING_FAILED, e.getMessage())); + } ); ActionListener markAsDeletingListener = ActionListener.wrap( @@ -192,10 +195,7 @@ protected void masterOperation(Task task, DeleteJobAction.Request request, Clust normalDeleteJob(parentTaskClient, request, finalListener); } }, - e -> { - auditor.error(request.getJobId(), Messages.getMessage(Messages.JOB_AUDIT_DELETING_FAILED, e.getMessage())); - finalListener.onFailure(e); - }); + finalListener::onFailure); ActionListener jobExistsListener = ActionListener.wrap( response -> { From eceb3185c79b9a777e6f53e899bc61b4f10ffd42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrique=20Gon=C3=A7alves?= <36671768+up201608320@users.noreply.github.com> Date: Tue, 29 Jan 2019 09:06:50 +0000 Subject: [PATCH 003/100] [ML] Make GetJobStats work with arbitrary wildcards and groups (#36683) The /_ml/anomaly_detectors/{job}/_stats endpoint now works correctly when {job} is a wildcard or job group. Closes #34745 --- .../elasticsearch/xpack/core/ml/action/GetJobsStatsAction.java | 2 +- .../xpack/ml/action/TransportGetJobsStatsActionTests.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetJobsStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetJobsStatsAction.java index dd3c4618f7025..23d320143ab34 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetJobsStatsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetJobsStatsAction.java @@ -118,7 +118,7 @@ public boolean allowNoJobs() { @Override public boolean match(Task task) { - return OpenJobAction.JobTaskMatcher.match(task, jobId); + return expandedJobsIds.stream().anyMatch(jobId -> OpenJobAction.JobTaskMatcher.match(task, jobId)); } @Override diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsActionTests.java index 2ee184ec877ed..eb5a5a3dda526 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsActionTests.java @@ -39,7 +39,7 @@ public void testDetermineJobIds() { result = determineJobIdsWithoutLiveStats(Arrays.asList("id1", "id2", "id3"), Collections.singletonList(new GetJobsStatsAction.Response.JobStats("id1", new DataCounts("id1"), null, null, - JobState.CLOSED, null, null, null)) + JobState.OPENED, null, null, null)) ); assertEquals(2, result.size()); assertEquals("id2", result.get(0)); From 827c4f656711a4a8897a5139be386119a33b5f69 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Tue, 29 Jan 2019 10:44:01 +0100 Subject: [PATCH 004/100] Make Version.java aware of 6.x Lucene upgrade Relates to #37913 --- server/src/main/java/org/elasticsearch/Version.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index 8f4d799713b09..b66630344a7ea 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -121,7 +121,7 @@ public class Version implements Comparable, ToXContentFragment { public static final int V_6_6_0_ID = 6060099; public static final Version V_6_6_0 = new Version(V_6_6_0_ID, org.apache.lucene.util.Version.LUCENE_7_6_0); public static final int V_6_7_0_ID = 6070099; - public static final Version V_6_7_0 = new Version(V_6_7_0_ID, org.apache.lucene.util.Version.LUCENE_7_6_0); + public static final Version V_6_7_0 = new Version(V_6_7_0_ID, org.apache.lucene.util.Version.LUCENE_7_7_0); public static final int V_7_0_0_ID = 7000099; public static final Version V_7_0_0 = new Version(V_7_0_0_ID, org.apache.lucene.util.Version.LUCENE_8_0_0); public static final Version CURRENT = V_7_0_0; From 504a89feaf7c7eed55e7c44e7606fbc9ca7dae9a Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Tue, 29 Jan 2019 12:43:04 +0100 Subject: [PATCH 005/100] Step down as master when configured out of voting configuration (#37802) Abdicates to another master-eligible node once the active master is reconfigured out of the voting configuration, for example through the use of voting configuration exclusions. Follow-up to #37712 --- .../discovery/adding-removing-nodes.asciidoc | 5 +- .../cluster/coordination/Coordinator.java | 47 ++++++++++++++++--- .../cluster/coordination/Publication.java | 16 +++++++ .../cluster/coordination/Reconfigurator.java | 29 +++++++++--- .../cluster/SpecificMasterNodesIT.java | 13 +++-- .../coordination/CoordinatorTests.java | 3 ++ .../coordination/PublicationTests.java | 4 ++ .../coordination/ReconfiguratorTests.java | 38 +++++++++++---- .../coordination/VotingConfigurationIT.java | 41 ++++++++++++++++ 9 files changed, 170 insertions(+), 26 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/cluster/coordination/VotingConfigurationIT.java diff --git a/docs/reference/modules/discovery/adding-removing-nodes.asciidoc b/docs/reference/modules/discovery/adding-removing-nodes.asciidoc index 3b416ea51d223..ccc0e99125371 100644 --- a/docs/reference/modules/discovery/adding-removing-nodes.asciidoc +++ b/docs/reference/modules/discovery/adding-removing-nodes.asciidoc @@ -72,7 +72,10 @@ The node that should be added to the exclusions list is specified using <> in place of `node_name` here. If a call to the voting configuration exclusions API fails, you can safely retry it. Only a successful response guarantees that the node has actually been removed from the -voting configuration and will not be reinstated. +voting configuration and will not be reinstated. If it's the active master that +was removed from the voting configuration, then it will abdicate to another +master-eligible node that's still in the voting configuration, if such a node +is available. Although the voting configuration exclusions API is most useful for down-scaling a two-node to a one-node cluster, it is also possible to use it to remove diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java index aabe5466d69a9..4bf977f8398ce 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java @@ -112,6 +112,7 @@ public class Coordinator extends AbstractLifecycleComponent implements Discovery private final PeerFinder peerFinder; private final PreVoteCollector preVoteCollector; + private final Random random; private final ElectionSchedulerFactory electionSchedulerFactory; private final UnicastConfiguredHostsResolver configuredHostsResolver; private final TimeValue publishTimeout; @@ -153,6 +154,7 @@ public Coordinator(String nodeName, Settings settings, ClusterSettings clusterSe this.lastJoin = Optional.empty(); this.joinAccumulator = new InitialJoinAccumulator(); this.publishTimeout = PUBLISH_TIMEOUT_SETTING.get(settings); + this.random = random; this.electionSchedulerFactory = new ElectionSchedulerFactory(settings, random, transportService.getThreadPool()); this.preVoteCollector = new PreVoteCollector(transportService, this::startElection, this::updateMaxTermSeen); configuredHostsResolver = new UnicastConfiguredHostsResolver(nodeName, settings, transportService, unicastHostsProvider); @@ -366,11 +368,33 @@ private void startElection() { } } + private void abdicateTo(DiscoveryNode newMaster) { + assert Thread.holdsLock(mutex); + assert mode == Mode.LEADER : "expected to be leader on abdication but was " + mode; + assert newMaster.isMasterNode() : "should only abdicate to master-eligible node but was " + newMaster; + final StartJoinRequest startJoinRequest = new StartJoinRequest(newMaster, Math.max(getCurrentTerm(), maxTermSeen) + 1); + logger.info("abdicating to {} with term {}", newMaster, startJoinRequest.getTerm()); + getLastAcceptedState().nodes().mastersFirstStream().forEach(node -> { + if (isZen1Node(node) == false) { + joinHelper.sendStartJoinRequest(startJoinRequest, node); + } + }); + // handling of start join messages on the local node will be dispatched to the generic thread-pool + assert mode == Mode.LEADER : "should still be leader after sending abdication messages " + mode; + // explicitly move node to candidate state so that the next cluster state update task yields an onNoLongerMaster event + becomeCandidate("after abdicating to " + newMaster); + } + private static boolean electionQuorumContainsLocalNode(ClusterState lastAcceptedState) { - final String localNodeId = lastAcceptedState.nodes().getLocalNodeId(); - assert localNodeId != null; - return lastAcceptedState.getLastCommittedConfiguration().getNodeIds().contains(localNodeId) - || lastAcceptedState.getLastAcceptedConfiguration().getNodeIds().contains(localNodeId); + final DiscoveryNode localNode = lastAcceptedState.nodes().getLocalNode(); + assert localNode != null; + return electionQuorumContains(lastAcceptedState, localNode); + } + + private static boolean electionQuorumContains(ClusterState lastAcceptedState, DiscoveryNode node) { + final String nodeId = node.getId(); + return lastAcceptedState.getLastCommittedConfiguration().getNodeIds().contains(nodeId) + || lastAcceptedState.getLastAcceptedConfiguration().getNodeIds().contains(nodeId); } private Optional ensureTermAtLeast(DiscoveryNode sourceNode, long targetTerm) { @@ -780,7 +804,7 @@ ClusterState improveConfiguration(ClusterState clusterState) { .filter(this::hasJoinVoteFrom).filter(discoveryNode -> isZen1Node(discoveryNode) == false).collect(Collectors.toSet()); final VotingConfiguration newConfig = reconfigurator.reconfigure(liveNodes, clusterState.getVotingConfigExclusions().stream().map(VotingConfigExclusion::getNodeId).collect(Collectors.toSet()), - clusterState.getLastAcceptedConfiguration()); + getLocalNode(), clusterState.getLastAcceptedConfiguration()); if (newConfig.equals(clusterState.getLastAcceptedConfiguration()) == false) { assert coordinationState.get().joinVotesHaveQuorumFor(newConfig); return ClusterState.builder(clusterState).metaData(MetaData.builder(clusterState.metaData()) @@ -1192,7 +1216,18 @@ public void onSuccess(String source) { updateMaxTermSeen(getCurrentTerm()); if (mode == Mode.LEADER) { - scheduleReconfigurationIfNeeded(); + final ClusterState state = getLastAcceptedState(); // committed state + if (electionQuorumContainsLocalNode(state) == false) { + final List masterCandidates = completedNodes().stream() + .filter(DiscoveryNode::isMasterNode) + .filter(node -> electionQuorumContains(state, node)) + .collect(Collectors.toList()); + if (masterCandidates.isEmpty() == false) { + abdicateTo(masterCandidates.get(random.nextInt(masterCandidates.size()))); + } + } else { + scheduleReconfigurationIfNeeded(); + } } lagDetector.startLagDetector(publishRequest.getAcceptedState().version()); } diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Publication.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Publication.java index 4aea820d6d9e0..da7c1d02a1e0b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/Publication.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Publication.java @@ -36,6 +36,7 @@ import java.util.Optional; import java.util.Set; import java.util.function.LongSupplier; +import java.util.stream.Collectors; public abstract class Publication { @@ -92,6 +93,13 @@ public void onFaultyNode(DiscoveryNode faultyNode) { onPossibleCompletion(); } + public List completedNodes() { + return publicationTargets.stream() + .filter(PublicationTarget::isSuccessfullyCompleted) + .map(PublicationTarget::getDiscoveryNode) + .collect(Collectors.toList()); + } + public boolean isCommitted() { return applyCommitRequest.isPresent(); } @@ -268,6 +276,10 @@ void onFaultyNode(DiscoveryNode faultyNode) { } } + DiscoveryNode getDiscoveryNode() { + return discoveryNode; + } + private void ackOnce(Exception e) { if (ackIsPending) { ackIsPending = false; @@ -280,6 +292,10 @@ boolean isActive() { && state != PublicationTargetState.APPLIED_COMMIT; } + boolean isSuccessfullyCompleted() { + return state == PublicationTargetState.APPLIED_COMMIT; + } + boolean isWaitingForQuorum() { return state == PublicationTargetState.WAITING_FOR_QUORUM; } diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Reconfigurator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Reconfigurator.java index 5c7b9562d8d8a..ebca37bdac0b1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/Reconfigurator.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Reconfigurator.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.util.set.Sets; import java.util.Collection; +import java.util.Collections; import java.util.Set; import java.util.TreeSet; import java.util.stream.Collectors; @@ -90,18 +91,23 @@ public String toString() { * @param retiredNodeIds Nodes that are leaving the cluster and which should not appear in the configuration if possible. Nodes that are * retired and not in the current configuration will never appear in the resulting configuration; this is useful * for shifting the vote in a 2-node cluster so one of the nodes can be restarted without harming availability. + * @param currentMaster The current master. Unless retired, we prefer to keep the current master in the config. * @param currentConfig The current configuration. As far as possible, we prefer to keep the current config as-is. * @return An optimal configuration, or leave the current configuration unchanged if the optimal configuration has no live quorum. */ - public VotingConfiguration reconfigure(Set liveNodes, Set retiredNodeIds, VotingConfiguration currentConfig) { + public VotingConfiguration reconfigure(Set liveNodes, Set retiredNodeIds, DiscoveryNode currentMaster, + VotingConfiguration currentConfig) { assert liveNodes.stream().noneMatch(Coordinator::isZen1Node) : liveNodes; - logger.trace("{} reconfiguring {} based on liveNodes={}, retiredNodeIds={}", this, currentConfig, liveNodes, retiredNodeIds); + assert liveNodes.contains(currentMaster) : "liveNodes = " + liveNodes + " master = " + currentMaster; + logger.trace("{} reconfiguring {} based on liveNodes={}, retiredNodeIds={}, currentMaster={}", + this, currentConfig, liveNodes, retiredNodeIds, currentMaster); /* * There are three true/false properties of each node in play: live/non-live, retired/non-retired and in-config/not-in-config. * Firstly we divide the nodes into disjoint sets based on these properties: * - * - nonRetiredInConfigNotLiveIds + * - nonRetiredMaster + * - nonRetiredNotMasterInConfigNotLiveIds * - nonRetiredInConfigLiveIds * - nonRetiredLiveNotInConfigIds * @@ -125,6 +131,17 @@ public VotingConfiguration reconfigure(Set liveNodes, Set final Set nonRetiredInConfigLiveIds = new TreeSet<>(liveInConfigIds); nonRetiredInConfigLiveIds.removeAll(retiredNodeIds); + final Set nonRetiredInConfigLiveMasterIds; + final Set nonRetiredInConfigLiveNotMasterIds; + if (nonRetiredInConfigLiveIds.contains(currentMaster.getId())) { + nonRetiredInConfigLiveNotMasterIds = new TreeSet<>(nonRetiredInConfigLiveIds); + nonRetiredInConfigLiveNotMasterIds.remove(currentMaster.getId()); + nonRetiredInConfigLiveMasterIds = Collections.singleton(currentMaster.getId()); + } else { + nonRetiredInConfigLiveNotMasterIds = nonRetiredInConfigLiveIds; + nonRetiredInConfigLiveMasterIds = Collections.emptySet(); + } + final Set nonRetiredLiveNotInConfigIds = Sets.sortedDifference(liveNodeIds, currentConfig.getNodeIds()); nonRetiredLiveNotInConfigIds.removeAll(retiredNodeIds); @@ -151,9 +168,9 @@ public VotingConfiguration reconfigure(Set liveNodes, Set * The new configuration is formed by taking this many nodes in the following preference order: */ final VotingConfiguration newConfig = new VotingConfiguration( - // live nodes first, preferring the current config, and if we need more then use non-live nodes - Stream.of(nonRetiredInConfigLiveIds, nonRetiredLiveNotInConfigIds, nonRetiredInConfigNotLiveIds) - .flatMap(Collection::stream).limit(targetSize).collect(Collectors.toSet())); + // live master first, then other live nodes, preferring the current config, and if we need more then use non-live nodes + Stream.of(nonRetiredInConfigLiveMasterIds, nonRetiredInConfigLiveNotMasterIds, nonRetiredLiveNotInConfigIds, + nonRetiredInConfigNotLiveIds).flatMap(Collection::stream).limit(targetSize).collect(Collectors.toSet())); if (newConfig.hasQuorum(liveNodeIds)) { return newConfig; diff --git a/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java b/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java index aaef1e58fb50e..aaf01b5e6e079 100644 --- a/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java @@ -31,12 +31,12 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; +import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.junit.annotations.TestLogging; import java.io.IOException; import java.util.Collections; import java.util.List; -import java.util.concurrent.ExecutionException; import static org.elasticsearch.discovery.zen.ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -106,7 +106,7 @@ public void testSimpleOnlyMasterNodeElection() throws IOException { .execute().actionGet().getState().nodes().getMasterNode().getName(), equalTo(nextMasterEligibleNodeName)); } - public void testElectOnlyBetweenMasterNodes() throws IOException, ExecutionException, InterruptedException { + public void testElectOnlyBetweenMasterNodes() throws Exception { logger.info("--> start data node / non master node"); internalCluster().startNode(Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), true) .put(Node.NODE_MASTER_SETTING.getKey(), false).put("discovery.initial_state_timeout", "1s")); @@ -138,7 +138,14 @@ public void testElectOnlyBetweenMasterNodes() throws IOException, ExecutionExcep logger.info("--> closing master node (1)"); client().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(new String[]{masterNodeName})).get(); - internalCluster().stopCurrentMasterNode(); + // removing the master from the voting configuration immediately triggers the master to step down + assertBusy(() -> { + assertThat(internalCluster().nonMasterClient().admin().cluster().prepareState() + .execute().actionGet().getState().nodes().getMasterNode().getName(), equalTo(nextMasterEligableNodeName)); + assertThat(internalCluster().masterClient().admin().cluster().prepareState() + .execute().actionGet().getState().nodes().getMasterNode().getName(), equalTo(nextMasterEligableNodeName)); + }); + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(masterNodeName)); assertThat(internalCluster().nonMasterClient().admin().cluster().prepareState() .execute().actionGet().getState().nodes().getMasterNode().getName(), equalTo(nextMasterEligableNodeName)); assertThat(internalCluster().masterClient().admin().cluster().prepareState() diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java index be40f0c888362..36495914bddec 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java @@ -66,6 +66,7 @@ import org.elasticsearch.test.disruption.DisruptableMockTransport.ConnectionStatus; import org.elasticsearch.transport.TransportService; import org.hamcrest.Matcher; +import org.hamcrest.core.IsCollectionContaining; import org.junit.After; import org.junit.Before; @@ -1331,6 +1332,8 @@ void stabilise(long stabilisationDurationMillis) { final VotingConfiguration lastCommittedConfiguration = lastAcceptedState.getLastCommittedConfiguration(); assertTrue(connectedNodeIds + " should be a quorum of " + lastCommittedConfiguration, lastCommittedConfiguration.hasQuorum(connectedNodeIds)); + assertThat("leader " + leader.getLocalNode() + " should be part of voting configuration " + lastCommittedConfiguration, + lastCommittedConfiguration.getNodeIds(), IsCollectionContaining.hasItem(leader.getLocalNode().getId())); assertThat("no reconfiguration is in progress", lastAcceptedState.getLastCommittedConfiguration(), equalTo(lastAcceptedState.getLastAcceptedConfiguration())); diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/PublicationTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/PublicationTests.java index 658250bc7a4da..d332888c185ac 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/PublicationTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/PublicationTests.java @@ -56,6 +56,7 @@ import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; public class PublicationTests extends ESTestCase { @@ -178,6 +179,7 @@ public void testSimpleClusterStatePublishing() throws InterruptedException { discoveryNodes, singleNodeConfig, singleNodeConfig, 42L), ackListener, Collections.emptySet()); assertThat(publication.pendingPublications.keySet(), equalTo(discoNodes)); + assertThat(publication.completedNodes(), empty()); assertTrue(publication.pendingCommits.isEmpty()); AtomicBoolean processedNode1PublishResponse = new AtomicBoolean(); boolean delayProcessingNode2PublishResponse = randomBoolean(); @@ -232,10 +234,12 @@ public void testSimpleClusterStatePublishing() throws InterruptedException { assertFalse(publication.completed); assertFalse(publication.committed); + assertThat(publication.completedNodes(), containsInAnyOrder(n1, n3)); publication.pendingCommits.get(n2).onResponse(TransportResponse.Empty.INSTANCE); } assertTrue(publication.completed); + assertThat(publication.completedNodes(), containsInAnyOrder(n1, n2, n3)); assertTrue(publication.committed); assertThat(ackListener.await(0L, TimeUnit.SECONDS), containsInAnyOrder(n1, n2, n3)); diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/ReconfiguratorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/ReconfiguratorTests.java index 7e7c7adbe1af9..bbd9514222c77 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/ReconfiguratorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/ReconfiguratorTests.java @@ -31,6 +31,7 @@ import org.junit.Before; import java.util.Arrays; +import java.util.Comparator; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -52,6 +53,7 @@ public void testReconfigurationExamples() { check(nodes("a"), conf("a"), true, conf("a")); check(nodes("a", "b"), conf("a"), true, conf("a")); + check(nodes("a", "b"), conf("b"), true, conf("b")); check(nodes("a", "b"), conf("a", "c"), true, conf("a")); check(nodes("a", "b"), conf("a", "b"), true, conf("a")); check(nodes("a", "b"), conf("a", "b", "e"), true, conf("a", "b", "e")); @@ -64,6 +66,7 @@ public void testReconfigurationExamples() { check(nodes("a", "b", "c", "d"), conf("a", "b", "e"), true, conf("a", "b", "c")); check(nodes("a", "b", "c", "d", "e"), conf("a", "f", "g"), true, conf("a", "b", "c", "d", "e")); check(nodes("a", "b", "c", "d"), conf("a", "b", "c", "d", "e"), true, conf("a", "b", "c")); + check(nodes("e", "a", "b", "c"), retired(), "e", conf("a", "b", "c", "d", "e"), true, conf("a", "b", "e")); check(nodes("a", "b", "c"), conf("a", "b", "c", "d", "e"), true, conf("a", "b", "c")); check(nodes("a"), conf("a"), false, conf("a")); @@ -124,7 +127,8 @@ public void testAutoShrinking() { final int quorumSize = Math.max(liveNodes.length / 2 + 1, initialVotingNodes.length < 3 ? 1 : 2); - final VotingConfiguration finalConfig = reconfigurator.reconfigure(liveNodesSet, emptySet(), initialConfig); + final VotingConfiguration finalConfig = reconfigurator.reconfigure(liveNodesSet, emptySet(), + randomFrom(liveNodesSet), initialConfig); final String description = "reconfigure " + liveNodesSet + " from " + initialConfig + " yielded " + finalConfig; @@ -152,7 +156,8 @@ public void testManualShrinking() { final int quorumSize = Math.max(liveNodes.length, initialVotingNodes.length) / 2 + 1; - final VotingConfiguration finalConfig = reconfigurator.reconfigure(liveNodesSet, emptySet(), initialConfig); + final VotingConfiguration finalConfig = reconfigurator.reconfigure(liveNodesSet, emptySet(), randomFrom(liveNodesSet), + initialConfig); final String description = "reconfigure " + liveNodesSet + " from " + initialConfig + " yielded " + finalConfig; @@ -187,13 +192,20 @@ private void check(Set liveNodes, VotingConfiguration config, boo private void check(Set liveNodes, Set retired, VotingConfiguration config, boolean autoShrinkVotingConfiguration, VotingConfiguration expectedConfig) { + final DiscoveryNode master = liveNodes.stream().sorted(Comparator.comparing(DiscoveryNode::getId)).findFirst().get(); + check(liveNodes, retired, master.getId(), config, autoShrinkVotingConfiguration, expectedConfig); + } + + private void check(Set liveNodes, Set retired, String masterId, VotingConfiguration config, + boolean autoShrinkVotingConfiguration, VotingConfiguration expectedConfig) { final Reconfigurator reconfigurator = makeReconfigurator(Settings.builder() .put(CLUSTER_AUTO_SHRINK_VOTING_CONFIGURATION.getKey(), autoShrinkVotingConfiguration) .build()); - final VotingConfiguration adaptedConfig = reconfigurator.reconfigure(liveNodes, retired, config); - assertEquals(new ParameterizedMessage("[liveNodes={}, retired={}, config={}, autoShrinkVotingConfiguration={}]", - liveNodes, retired, config, autoShrinkVotingConfiguration).getFormattedMessage(), + final DiscoveryNode master = liveNodes.stream().filter(n -> n.getId().equals(masterId)).findFirst().get(); + final VotingConfiguration adaptedConfig = reconfigurator.reconfigure(liveNodes, retired, master, config); + assertEquals(new ParameterizedMessage("[liveNodes={}, retired={}, master={}, config={}, autoShrinkVotingConfiguration={}]", + liveNodes, retired, master, config, autoShrinkVotingConfiguration).getFormattedMessage(), expectedConfig, adaptedConfig); } @@ -206,18 +218,24 @@ public void testDynamicSetting() { final Reconfigurator reconfigurator = new Reconfigurator(Settings.EMPTY, clusterSettings); final VotingConfiguration initialConfig = conf("a", "b", "c", "d", "e"); + Set twoNodes = nodes("a", "b"); + Set threeNodes = nodes("a", "b", "c"); + // default is "true" - assertThat(reconfigurator.reconfigure(nodes("a", "b"), retired(), initialConfig), equalTo(conf("a", "b", "c"))); + assertThat(reconfigurator.reconfigure(twoNodes, retired(), randomFrom(twoNodes), initialConfig), equalTo(conf("a", "b", "c"))); // update to "false" clusterSettings.applySettings(Settings.builder().put(CLUSTER_AUTO_SHRINK_VOTING_CONFIGURATION.getKey(), "false").build()); - assertThat(reconfigurator.reconfigure(nodes("a", "b"), retired(), initialConfig), sameInstance(initialConfig)); // no quorum - assertThat(reconfigurator.reconfigure(nodes("a", "b", "c"), retired(), initialConfig), equalTo(conf("a", "b", "c", "d", "e"))); - assertThat(reconfigurator.reconfigure(nodes("a", "b", "c"), retired("d"), initialConfig), equalTo(conf("a", "b", "c", "e"))); + assertThat(reconfigurator.reconfigure(twoNodes, retired(), randomFrom(twoNodes), initialConfig), + sameInstance(initialConfig)); // no quorum + assertThat(reconfigurator.reconfigure(threeNodes, retired(), randomFrom(threeNodes), initialConfig), + equalTo(conf("a", "b", "c", "d", "e"))); + assertThat(reconfigurator.reconfigure(threeNodes, retired("d"), randomFrom(threeNodes), initialConfig), + equalTo(conf("a", "b", "c", "e"))); // explicitly set to "true" clusterSettings.applySettings(Settings.builder().put(CLUSTER_AUTO_SHRINK_VOTING_CONFIGURATION.getKey(), "true").build()); - assertThat(reconfigurator.reconfigure(nodes("a", "b"), retired(), initialConfig), equalTo(conf("a", "b", "c"))); + assertThat(reconfigurator.reconfigure(twoNodes, retired(), randomFrom(twoNodes), initialConfig), equalTo(conf("a", "b", "c"))); expectThrows(IllegalArgumentException.class, () -> clusterSettings.applySettings(Settings.builder().put(CLUSTER_AUTO_SHRINK_VOTING_CONFIGURATION.getKey(), "blah").build())); diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/VotingConfigurationIT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/VotingConfigurationIT.java new file mode 100644 index 0000000000000..8c6775cb6c91e --- /dev/null +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/VotingConfigurationIT.java @@ -0,0 +1,41 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.cluster.coordination; + +import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsAction; +import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest; +import org.elasticsearch.common.Priority; +import org.elasticsearch.test.ESIntegTestCase; + +import java.util.concurrent.ExecutionException; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) +public class VotingConfigurationIT extends ESIntegTestCase { + + public void testAbdicateAfterVotingConfigExclusionAdded() throws ExecutionException, InterruptedException { + internalCluster().startNodes(2); + final String originalMaster = internalCluster().getMasterName(); + + logger.info("--> excluding master node {}", originalMaster); + client().execute(AddVotingConfigExclusionsAction.INSTANCE, + new AddVotingConfigExclusionsRequest(new String[]{originalMaster})).get(); + client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).get(); + assertNotEquals(originalMaster, internalCluster().getMasterName()); + } +} From 460f10ce602f067816c2615f035a3d42f42d3ed1 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Tue, 29 Jan 2019 13:15:58 +0100 Subject: [PATCH 006/100] Close Index API should force a flush if a sync is needed (#37961) This commit changes the TransportVerifyShardBeforeCloseAction so that it issues a forced flush, forcing the translog and the Lucene commit to contain the same max seq number and global checkpoint in the case the Translog contains operations that were not written in the IndexWriter (like a Delete that touches a non existing doc). This way the assertion added in #37426 won't trip. Related to #33888 --- ...TransportVerifyShardBeforeCloseAction.java | 9 ++++-- .../index/engine/FrozenIndexTests.java | 30 +++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportVerifyShardBeforeCloseAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportVerifyShardBeforeCloseAction.java index f08f6ea7dffa2..a36a012f397ec 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportVerifyShardBeforeCloseAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportVerifyShardBeforeCloseAction.java @@ -18,6 +18,8 @@ */ package org.elasticsearch.action.admin.indices.close; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.flush.FlushRequest; import org.elasticsearch.action.support.ActionFilters; @@ -50,6 +52,7 @@ public class TransportVerifyShardBeforeCloseAction extends TransportReplicationA TransportVerifyShardBeforeCloseAction.ShardRequest, TransportVerifyShardBeforeCloseAction.ShardRequest, ReplicationResponse> { public static final String NAME = CloseIndexAction.NAME + "[s]"; + protected Logger logger = LogManager.getLogger(getClass()); @Inject public TransportVerifyShardBeforeCloseAction(final Settings settings, final TransportService transportService, @@ -111,8 +114,10 @@ private void executeShardOperation(final ShardRequest request, final IndexShard throw new IllegalStateException("Global checkpoint [" + indexShard.getGlobalCheckpoint() + "] mismatches maximum sequence number [" + maxSeqNo + "] on index shard " + shardId); } - indexShard.flush(new FlushRequest()); - logger.debug("{} shard is ready for closing", shardId); + + final boolean forced = indexShard.isSyncNeeded(); + indexShard.flush(new FlushRequest().force(forced)); + logger.trace("{} shard is ready for closing [forced:{}]", shardId, forced); } @Override diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java index c0493b6efd1fe..094c79efb50a9 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java @@ -8,6 +8,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.support.IndicesOptions; @@ -26,6 +27,7 @@ import org.elasticsearch.index.shard.IndexShardTestCase; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchService; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.internal.AliasFilter; @@ -46,6 +48,8 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.is; public class FrozenIndexTests extends ESSingleNodeTestCase { @@ -340,4 +344,30 @@ public void testFreezeIndexIncreasesIndexSettingsVersion() throws ExecutionExcep assertThat(client().admin().cluster().prepareState().get().getState().metaData().index(index).getSettingsVersion(), equalTo(settingsVersion + 1)); } + + public void testFreezeEmptyIndexWithTranslogOps() throws Exception { + final String indexName = "empty"; + createIndex(indexName, Settings.builder() + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .put("index.refresh_interval", TimeValue.MINUS_ONE) + .build()); + + final long nbNoOps = randomIntBetween(1, 10); + for (long i = 0; i < nbNoOps; i++) { + final DeleteResponse deleteResponse = client().prepareDelete(indexName, "_doc", Long.toString(i)).get(); + assertThat(deleteResponse.status(), is(RestStatus.NOT_FOUND)); + } + + final IndicesService indicesService = getInstanceFromNode(IndicesService.class); + assertBusy(() -> { + final Index index = client().admin().cluster().prepareState().get().getState().metaData().index(indexName).getIndex(); + final IndexService indexService = indicesService.indexService(index); + assertThat(indexService.hasShard(0), is(true)); + assertThat(indexService.getShard(0).getGlobalCheckpoint(), greaterThanOrEqualTo(nbNoOps - 1L)); + }); + + assertAcked(new XPackClient(client()).freeze(new TransportFreezeIndexAction.FreezeRequest(indexName))); + assertIndexFrozen(indexName); + } } From 4f4113e96474efde4ebeee351ff947ae22f6723d Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Tue, 29 Jan 2019 14:53:55 +0100 Subject: [PATCH 007/100] Rename security audit.log to _audit.json (#37916) in order to keep json logs consistent the security audit logs are renamed from .log to .json relates #32850 --- docs/reference/settings/audit-settings.asciidoc | 2 +- x-pack/docs/en/security/auditing/event-types.asciidoc | 2 +- x-pack/docs/en/security/auditing/output-logfile.asciidoc | 8 ++++---- x-pack/docs/en/security/auditing/overview.asciidoc | 2 +- x-pack/docs/en/security/configuring-es.asciidoc | 2 +- x-pack/plugin/core/src/main/config/log4j2.properties | 4 ++-- x-pack/plugin/sql/qa/security/build.gradle | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/reference/settings/audit-settings.asciidoc b/docs/reference/settings/audit-settings.asciidoc index 483c889ce5898..d4762d9f42fb1 100644 --- a/docs/reference/settings/audit-settings.asciidoc +++ b/docs/reference/settings/audit-settings.asciidoc @@ -14,7 +14,7 @@ file. For more information, see `xpack.security.audit.enabled`:: Set to `true` to enable auditing on the node. The default value is `false`. -This puts the auditing events in a dedicated file named `_audit.log` +This puts the auditing events in a dedicated file named `_audit.json` on each node. For more information, see <>. [[event-audit-settings]] diff --git a/x-pack/docs/en/security/auditing/event-types.asciidoc b/x-pack/docs/en/security/auditing/event-types.asciidoc index ace4d7d4c09c8..19947e40b5553 100644 --- a/x-pack/docs/en/security/auditing/event-types.asciidoc +++ b/x-pack/docs/en/security/auditing/event-types.asciidoc @@ -48,7 +48,7 @@ The following is a list of the events that can be generated: In 6.5.0, there is a new <> format. This format also brings in a few changes for audit event attributes. -The new format is output to the `_audit.log` file. +The new format is output to the `_audit.json` file. The audit entries are formatted as flat JSON documents (that is to say, no nested objects), one per line. Hence, the attribute names are JSON keys and they follow a dotted name syntax. Any attributes that lack a value (`null`) are not diff --git a/x-pack/docs/en/security/auditing/output-logfile.asciidoc b/x-pack/docs/en/security/auditing/output-logfile.asciidoc index ac7128852b942..f5b1dbad79ae9 100644 --- a/x-pack/docs/en/security/auditing/output-logfile.asciidoc +++ b/x-pack/docs/en/security/auditing/output-logfile.asciidoc @@ -3,7 +3,7 @@ === Logfile audit output The `logfile` audit output is the default output for auditing. It writes data to -the `_audit.log` file in the logs directory. To maintain +the `_audit.json` file in the logs directory. To maintain compatibility with releases prior to 6.5.0, a `_access.log` file is also generated. They differ in the output format but the contents are similar. For systems that are not ingesting the audit file for search or @@ -43,7 +43,7 @@ by default points to the `elasticsearch.log` file. [[audit-log-entry-format]] === Log entry format -The log entries in the `_audit.log` file have the following format: +The log entries in the `_audit.json` file have the following format: - Each log entry is a one line JSON document and each one is printed on a separate line. - The fields of a log entry are ordered. However, if a field does not have a value it @@ -100,14 +100,14 @@ audited in plain text when including the request body in audit events. [[logging-file]] You can also configure how the logfile is written in the `log4j2.properties` file located in `ES_PATH_CONF`. By default, audit information is appended to the -`_audit.log` file located in the standard Elasticsearch `logs` directory +`_audit.json` file located in the standard Elasticsearch `logs` directory (typically located at `$ES_HOME/logs`). The file rolls over on a daily basis. The deprecated logfile audit format (`_access.log`) can be disabled from the same `log4j2.properties` file (hint: look for the comment instructing to set the log level to `off`). The deprecated format is a duplication of information that is in place to assure backwards compatibility. If you are not strict about the audit format it is strongly recommended to only use the -`_audit.log` log appender. +`_audit.json` log appender. [float] [[audit-log-ignore-policy]] diff --git a/x-pack/docs/en/security/auditing/overview.asciidoc b/x-pack/docs/en/security/auditing/overview.asciidoc index f0b58684e078f..51f24c9bcced0 100644 --- a/x-pack/docs/en/security/auditing/overview.asciidoc +++ b/x-pack/docs/en/security/auditing/overview.asciidoc @@ -13,5 +13,5 @@ Audit logs are **disabled** by default. To enable this functionality, you must set `xpack.security.audit.enabled` to `true` in `elasticsearch.yml`. ============================================================================ -The audit log persists events to a dedicated `_audit.log` file on +The audit log persists events to a dedicated `_audit.json` file on the host's file system (on each node). diff --git a/x-pack/docs/en/security/configuring-es.asciidoc b/x-pack/docs/en/security/configuring-es.asciidoc index db2c8c664b9d2..e1dab76293c5f 100644 --- a/x-pack/docs/en/security/configuring-es.asciidoc +++ b/x-pack/docs/en/security/configuring-es.asciidoc @@ -131,7 +131,7 @@ and <>. .. Restart {es}. -Events are logged to a dedicated `_audit.log` file in +Events are logged to a dedicated `_audit.json` file in `ES_HOME/logs`, on each cluster node. -- diff --git a/x-pack/plugin/core/src/main/config/log4j2.properties b/x-pack/plugin/core/src/main/config/log4j2.properties index 52b6ce3950690..2b7e112eb14fc 100644 --- a/x-pack/plugin/core/src/main/config/log4j2.properties +++ b/x-pack/plugin/core/src/main/config/log4j2.properties @@ -1,6 +1,6 @@ appender.audit_rolling.type = RollingFile appender.audit_rolling.name = audit_rolling -appender.audit_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_audit.log +appender.audit_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_audit.json appender.audit_rolling.layout.type = PatternLayout appender.audit_rolling.layout.pattern = {\ "@timestamp":"%d{ISO8601}"\ @@ -64,7 +64,7 @@ appender.audit_rolling.layout.pattern = {\ # "rule" name of the applied rulee if the "origin.type" is "ip_filter" # "event.category" fixed value "elasticsearch-audit" -appender.audit_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_audit-%d{yyyy-MM-dd}.log +appender.audit_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_audit-%d{yyyy-MM-dd}.json appender.audit_rolling.policies.type = Policies appender.audit_rolling.policies.time.type = TimeBasedTriggeringPolicy appender.audit_rolling.policies.time.interval = 1 diff --git a/x-pack/plugin/sql/qa/security/build.gradle b/x-pack/plugin/sql/qa/security/build.gradle index 69389b47accec..d9bffd393641d 100644 --- a/x-pack/plugin/sql/qa/security/build.gradle +++ b/x-pack/plugin/sql/qa/security/build.gradle @@ -42,7 +42,7 @@ subprojects { integTestRunner { systemProperty 'tests.audit.logfile', - "${ -> integTest.nodes[0].homeDir}/logs/${ -> integTest.nodes[0].clusterName }_audit.log" + "${ -> integTest.nodes[0].homeDir}/logs/${ -> integTest.nodes[0].clusterName }_audit.json" } runqa { From a6d4838a6779b4073008dc1f2e75a3ffb7cdfe0f Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Tue, 29 Jan 2019 14:56:22 +0100 Subject: [PATCH 008/100] Clean up allowPartialSearchResults serialization (#37911) When serializing allowPartialSearchResults to the shards through ShardSearchTransportRequest, we use an optional boolean field, though the corresponding instance member is declared `boolean` which can never be null. We also have an assert to verify that the incoming search request provides a non-null value for the flag, and a comment explaining that null should be considered a bug. This commit makes the allowPartialSearchResults method in ShardSearchRequest return a `boolean` rather than a `Boolean` and changes the serialization from optional to non optional, in a bw comp manner. --- .../search/internal/ShardSearchLocalRequest.java | 11 +++++++---- .../search/internal/ShardSearchRequest.java | 2 +- .../internal/ShardSearchTransportRequest.java | 2 +- .../elasticsearch/index/SearchSlowLogTests.java | 4 ++-- .../internal/ShardSearchTransportRequestTests.java | 14 ++++++++++++++ .../search/slice/SliceBuilderTests.java | 4 ++-- 6 files changed, 27 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java index 0921681124e33..08cdd2fc0dc16 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java @@ -170,7 +170,7 @@ public Boolean requestCache() { } @Override - public Boolean allowPartialSearchResults() { + public boolean allowPartialSearchResults() { return allowPartialSearchResults; } @@ -216,7 +216,9 @@ protected void innerReadFrom(StreamInput in) throws IOException { nowInMillis = in.readVLong(); requestCache = in.readOptionalBoolean(); clusterAlias = in.readOptionalString(); - if (in.getVersion().onOrAfter(Version.V_6_3_0)) { + if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + allowPartialSearchResults = in.readBoolean(); + } else if (in.getVersion().onOrAfter(Version.V_6_3_0)) { allowPartialSearchResults = in.readOptionalBoolean(); } if (in.getVersion().onOrAfter(Version.V_6_4_0)) { @@ -244,7 +246,9 @@ protected void innerWriteTo(StreamOutput out, boolean asKey) throws IOException } out.writeOptionalBoolean(requestCache); out.writeOptionalString(clusterAlias); - if (out.getVersion().onOrAfter(Version.V_6_3_0)) { + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + out.writeBoolean(allowPartialSearchResults); + } else if (out.getVersion().onOrAfter(Version.V_6_3_0)) { out.writeOptionalBoolean(allowPartialSearchResults); } if (asKey == false) { @@ -295,5 +299,4 @@ public Rewriteable rewrite(QueryRewriteContext ctx) throws IOException { } } } - } diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java index 3fc16584eb0bf..85800a8066c38 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java @@ -69,7 +69,7 @@ public interface ShardSearchRequest { Boolean requestCache(); - Boolean allowPartialSearchResults(); + boolean allowPartialSearchResults(); Scroll scroll(); diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java index 59d1c2e089e02..d1ff306e3c848 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java @@ -150,7 +150,7 @@ public Boolean requestCache() { } @Override - public Boolean allowPartialSearchResults() { + public boolean allowPartialSearchResults() { return shardSearchLocalRequest.allowPartialSearchResults(); } diff --git a/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java b/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java index fc24bdf9691de..6c510b3a9bd70 100644 --- a/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java +++ b/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java @@ -114,8 +114,8 @@ public Boolean requestCache() { } @Override - public Boolean allowPartialSearchResults() { - return null; + public boolean allowPartialSearchResults() { + return true; } @Override diff --git a/server/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java b/server/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java index da987a657260a..5cf7880ee32b7 100644 --- a/server/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java +++ b/server/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java @@ -40,6 +40,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.InvalidAliasNameException; import org.elasticsearch.search.AbstractSearchTestCase; +import org.elasticsearch.test.VersionUtils; import java.io.IOException; @@ -76,6 +77,19 @@ public void testSerialization() throws Exception { assertEquals(deserializedRequest.getAliasFilter(), shardSearchTransportRequest.getAliasFilter()); assertEquals(deserializedRequest.indexBoost(), shardSearchTransportRequest.indexBoost(), 0.0f); assertEquals(deserializedRequest.getClusterAlias(), shardSearchTransportRequest.getClusterAlias()); + assertEquals(shardSearchTransportRequest.allowPartialSearchResults(), deserializedRequest.allowPartialSearchResults()); + } + + public void testAllowPartialResultsSerializationPre7_0_0() throws IOException { + Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, VersionUtils.getPreviousVersion(Version.V_7_0_0)); + ShardSearchTransportRequest shardSearchTransportRequest = createShardSearchTransportRequest(); + ShardSearchTransportRequest deserializedRequest = + copyWriteable(shardSearchTransportRequest, namedWriteableRegistry, ShardSearchTransportRequest::new, version); + if (version.before(Version.V_6_3_0)) { + assertFalse(deserializedRequest.allowPartialSearchResults()); + } else { + assertEquals(shardSearchTransportRequest.allowPartialSearchResults(), deserializedRequest.allowPartialSearchResults()); + } } private ShardSearchTransportRequest createShardSearchTransportRequest() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java b/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java index 30ed0cb5ab5b5..04fd5947e5f6d 100644 --- a/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java @@ -163,8 +163,8 @@ public Boolean requestCache() { } @Override - public Boolean allowPartialSearchResults() { - return null; + public boolean allowPartialSearchResults() { + return true; } @Override From 42eec55837f99167b36ebc2dbffb3d6bd52d7279 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Tue, 29 Jan 2019 14:57:33 +0100 Subject: [PATCH 009/100] Replace failure.get().addSuppressed with failure.accumulateAndGet() (#37649) Also add a test for concurrent incoming failures --- .../action/support/GroupedActionListener.java | 5 +- .../support/GroupedActionListenerTests.java | 52 +++++++++++++------ 2 files changed, 39 insertions(+), 18 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/support/GroupedActionListener.java b/server/src/main/java/org/elasticsearch/action/support/GroupedActionListener.java index ed9b7c8d15d60..532396ee6095c 100644 --- a/server/src/main/java/org/elasticsearch/action/support/GroupedActionListener.java +++ b/server/src/main/java/org/elasticsearch/action/support/GroupedActionListener.java @@ -72,7 +72,10 @@ public void onResponse(T element) { @Override public void onFailure(Exception e) { if (failure.compareAndSet(null, e) == false) { - failure.get().addSuppressed(e); + failure.accumulateAndGet(e, (previous, current) -> { + previous.addSuppressed(current); + return previous; + }); } if (countDown.countDown()) { delegate.onFailure(failure.get()); diff --git a/server/src/test/java/org/elasticsearch/action/support/GroupedActionListenerTests.java b/server/src/test/java/org/elasticsearch/action/support/GroupedActionListenerTests.java index 2af2da7ba0939..9f6454d4e4bcf 100644 --- a/server/src/test/java/org/elasticsearch/action/support/GroupedActionListenerTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/GroupedActionListenerTests.java @@ -26,10 +26,14 @@ import java.util.Collection; import java.util.Collections; import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import static org.hamcrest.CoreMatchers.instanceOf; + public class GroupedActionListenerTests extends ESTestCase { public void testNotifications() throws InterruptedException { @@ -55,20 +59,17 @@ public void onFailure(Exception e) { Thread[] threads = new Thread[numThreads]; CyclicBarrier barrier = new CyclicBarrier(numThreads); for (int i = 0; i < numThreads; i++) { - threads[i] = new Thread() { - @Override - public void run() { - try { - barrier.await(10, TimeUnit.SECONDS); - } catch (Exception e) { - throw new AssertionError(e); - } - int c = 0; - while((c = count.incrementAndGet()) <= groupSize) { - listener.onResponse(c-1); - } + threads[i] = new Thread(() -> { + try { + barrier.await(10, TimeUnit.SECONDS); + } catch (Exception e) { + throw new AssertionError(e); + } + int c = 0; + while((c = count.incrementAndGet()) <= groupSize) { + listener.onResponse(c-1); } - }; + }); threads[i].start(); } for (Thread t : threads) { @@ -100,11 +101,9 @@ public void onFailure(Exception e) { excRef.set(e); } }; - Collection defaults = randomBoolean() ? Collections.singletonList(-1) : - Collections.emptyList(); + Collection defaults = randomBoolean() ? Collections.singletonList(-1) : Collections.emptyList(); int size = randomIntBetween(3, 4); - GroupedActionListener listener = new GroupedActionListener<>(result, size, - defaults); + GroupedActionListener listener = new GroupedActionListener<>(result, size, defaults); listener.onResponse(0); IOException ioException = new IOException(); RuntimeException rtException = new RuntimeException(); @@ -121,4 +120,23 @@ public void onFailure(Exception e) { listener.onResponse(1); assertNull(resRef.get()); } + + public void testConcurrentFailures() throws InterruptedException { + AtomicReference finalException = new AtomicReference<>(); + int numGroups = randomIntBetween(10, 100); + GroupedActionListener listener = new GroupedActionListener<>( + ActionListener.wrap(r -> {}, finalException::set), numGroups, Collections.emptyList()); + ExecutorService executorService = Executors.newFixedThreadPool(numGroups); + for (int i = 0; i < numGroups; i++) { + executorService.submit(() -> listener.onFailure(new IOException())); + } + + executorService.shutdown(); + executorService.awaitTermination(10, TimeUnit.SECONDS); + + Exception exception = finalException.get(); + assertNotNull(exception); + assertThat(exception, instanceOf(IOException.class)); + assertEquals(numGroups - 1, exception.getSuppressed().length); + } } From 2325fb9cb3530607475233e7c7281e5ff7241001 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Tue, 29 Jan 2019 14:58:11 +0100 Subject: [PATCH 010/100] Remove test only SearchShardTarget constructor (#37912) Remove SearchShardTarget test only constructor and replace all the usages with calls to the other constructor that accepts a ShardId. --- .../DiscountedCumulativeGainTests.java | 9 ++--- .../index/rankeval/EvalQueryQualityTests.java | 5 +-- .../rankeval/ExpectedReciprocalRankTests.java | 5 +-- .../rankeval/MeanReciprocalRankTests.java | 5 +-- .../index/rankeval/PrecisionAtKTests.java | 9 ++--- .../index/rankeval/RankEvalResponseTests.java | 7 ++-- .../search/SearchShardTarget.java | 6 ---- .../ElasticsearchExceptionTests.java | 26 +++++++-------- .../ExceptionSerializationTests.java | 3 +- .../action/search/CountedCollectorTests.java | 9 ++--- .../action/search/DfsQueryPhaseTests.java | 33 ++++++++++--------- .../action/search/FetchSearchPhaseTests.java | 26 +++++++++------ .../search/SearchPhaseControllerTests.java | 26 +++++++++------ .../SearchPhaseExecutionExceptionTests.java | 13 ++++---- .../search/SearchScrollAsyncActionTests.java | 11 ++++--- .../rest/BytesRestResponseTests.java | 7 ++-- .../SignificanceHeuristicTests.java | 6 ++-- .../search/MockSearchServiceTests.java | 5 +-- .../xpack/watcher/WatcherServiceTests.java | 3 +- .../CompareConditionSearchTests.java | 5 +-- .../execution/TriggeredWatchStoreTests.java | 5 +-- 21 files changed, 124 insertions(+), 100 deletions(-) diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java index 468a1ac2e5721..0e2d2b482e0b4 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.rankeval; +import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -30,7 +31,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.index.Index; +import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.test.ESTestCase; @@ -73,7 +74,7 @@ public void testDCGAt() { for (int i = 0; i < 6; i++) { rated.add(new RatedDocument("index", Integer.toString(i), relevanceRatings[i])); hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); - hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null)); + hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null, OriginalIndices.NONE)); } DiscountedCumulativeGain dcg = new DiscountedCumulativeGain(); assertEquals(EXPECTED_DCG, dcg.evaluate("id", hits, rated).metricScore(), DELTA); @@ -123,7 +124,7 @@ public void testDCGAtSixMissingRatings() { } } hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); - hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null)); + hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null, OriginalIndices.NONE)); } DiscountedCumulativeGain dcg = new DiscountedCumulativeGain(); EvalQueryQuality result = dcg.evaluate("id", hits, rated); @@ -180,7 +181,7 @@ public void testDCGAtFourMoreRatings() { SearchHit[] hits = new SearchHit[4]; for (int i = 0; i < 4; i++) { hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); - hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null)); + hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null, OriginalIndices.NONE)); } DiscountedCumulativeGain dcg = new DiscountedCumulativeGain(); EvalQueryQuality result = dcg.evaluate("id", hits, ratedDocs); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java index 7424542ac26aa..bfb3e3d55158f 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java @@ -19,13 +19,14 @@ package org.elasticsearch.index.rankeval; +import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.Index; +import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.test.ESTestCase; @@ -56,7 +57,7 @@ public static EvalQueryQuality randomEvalQueryQuality() { for (int i = 0; i < numberOfSearchHits; i++) { RatedSearchHit ratedSearchHit = RatedSearchHitTests.randomRatedSearchHit(); // we need to associate each hit with an index name otherwise rendering will not work - ratedSearchHit.getSearchHit().shard(new SearchShardTarget("_na_", new Index("index", "_na_"), 0, null)); + ratedSearchHit.getSearchHit().shard(new SearchShardTarget("_na_", new ShardId("index", "_na_", 0), null, OriginalIndices.NONE)); ratedHits.add(ratedSearchHit); } EvalQueryQuality evalQueryQuality = new EvalQueryQuality(randomAlphaOfLength(10), diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRankTests.java index fe33c246f7d7a..3906b05ce9d60 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRankTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.rankeval; +import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.text.Text; @@ -29,7 +30,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.index.Index; +import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.test.ESTestCase; @@ -116,7 +117,7 @@ private SearchHit[] createSearchHits(List rated, Integer[] releva rated.add(new RatedDocument("index", Integer.toString(i), relevanceRatings[i])); } hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); - hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null)); + hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null, OriginalIndices.NONE)); } return hits; } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java index fdb64806d5c9e..6b37fee3c5e8a 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.rankeval; +import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.text.Text; @@ -29,7 +30,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.index.Index; +import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.test.ESTestCase; @@ -204,7 +205,7 @@ private static SearchHit[] createSearchHits(int from, int to, String index) { SearchHit[] hits = new SearchHit[to + 1 - from]; for (int i = from; i <= to; i++) { hits[i] = new SearchHit(i, i + "", new Text(""), Collections.emptyMap()); - hits[i].shard(new SearchShardTarget("testnode", new Index(index, "uuid"), 0, null)); + hits[i].shard(new SearchShardTarget("testnode", new ShardId(index, "uuid", 0), null, OriginalIndices.NONE)); } return hits; } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java index b9e7bf25aaf7a..d24d8bc56d4eb 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.rankeval; +import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.text.Text; @@ -29,7 +30,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.index.Index; +import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.test.ESTestCase; @@ -111,7 +112,7 @@ public void testIgnoreUnlabeled() { // add an unlabeled search hit SearchHit[] searchHits = Arrays.copyOf(toSearchHits(rated, "test"), 3); searchHits[2] = new SearchHit(2, "2", new Text("testtype"), Collections.emptyMap()); - searchHits[2].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null)); + searchHits[2].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null, OriginalIndices.NONE)); EvalQueryQuality evaluated = (new PrecisionAtK()).evaluate("id", searchHits, rated); assertEquals((double) 2 / 3, evaluated.metricScore(), 0.00001); @@ -130,7 +131,7 @@ public void testNoRatedDocs() throws Exception { SearchHit[] hits = new SearchHit[5]; for (int i = 0; i < 5; i++) { hits[i] = new SearchHit(i, i + "", new Text("type"), Collections.emptyMap()); - hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null)); + hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null, OriginalIndices.NONE)); } EvalQueryQuality evaluated = (new PrecisionAtK()).evaluate("id", hits, Collections.emptyList()); assertEquals(0.0d, evaluated.metricScore(), 0.00001); @@ -252,7 +253,7 @@ private static SearchHit[] toSearchHits(List rated, String index) SearchHit[] hits = new SearchHit[rated.size()]; for (int i = 0; i < rated.size(); i++) { hits[i] = new SearchHit(i, i + "", new Text(""), Collections.emptyMap()); - hits[i].shard(new SearchShardTarget("testnode", new Index(index, "uuid"), 0, null)); + hits[i].shard(new SearchShardTarget("testnode", new ShardId(index, "uuid", 0), null, OriginalIndices.NONE)); } return hits; } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java index c12f923aa6089..070b2439e53cb 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.cluster.block.ClusterBlockException; @@ -37,7 +38,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.discovery.DiscoverySettings; -import org.elasticsearch.index.Index; +import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchParseException; import org.elasticsearch.search.SearchShardTarget; @@ -69,7 +70,7 @@ public class RankEvalResponseTests extends ESTestCase { new IllegalArgumentException("Closed resource", new RuntimeException("Resource")), new SearchPhaseExecutionException("search", "all shards failed", new ShardSearchFailure[] { new ShardSearchFailure(new ParsingException(1, 2, "foobar", null), - new SearchShardTarget("node_1", new Index("foo", "_na_"), 1, null)) }), + new SearchShardTarget("node_1", new ShardId("foo", "_na_", 1), null, OriginalIndices.NONE)) }), new ElasticsearchException("Parsing failed", new ParsingException(9, 42, "Wrong state", new NullPointerException("Unexpected null value"))) }; @@ -181,7 +182,7 @@ public void testToXContent() throws IOException { private static RatedSearchHit searchHit(String index, int docId, Integer rating) { SearchHit hit = new SearchHit(docId, docId + "", new Text(""), Collections.emptyMap()); - hit.shard(new SearchShardTarget("testnode", new Index(index, "uuid"), 0, null)); + hit.shard(new SearchShardTarget("testnode", new ShardId(index, "uuid", 0), null, OriginalIndices.NONE)); hit.score(1.0f); return new RatedSearchHit(hit, rating != null ? OptionalInt.of(rating) : OptionalInt.empty()); } diff --git a/server/src/main/java/org/elasticsearch/search/SearchShardTarget.java b/server/src/main/java/org/elasticsearch/search/SearchShardTarget.java index 42f3b67e358e4..6aadf8425997d 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchShardTarget.java +++ b/server/src/main/java/org/elasticsearch/search/SearchShardTarget.java @@ -25,7 +25,6 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.text.Text; -import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.transport.RemoteClusterAware; @@ -62,11 +61,6 @@ public SearchShardTarget(String nodeId, ShardId shardId, @Nullable String cluste this.clusterAlias = clusterAlias; } - //this constructor is only used in tests - public SearchShardTarget(String nodeId, Index index, int shardId, String clusterAlias) { - this(nodeId, new ShardId(index, shardId), clusterAlias, OriginalIndices.NONE); - } - @Nullable public String getNodeId() { return nodeId.string(); diff --git a/server/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java b/server/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java index 4faf754b13869..21ee15d01cf81 100644 --- a/server/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java +++ b/server/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java @@ -115,9 +115,9 @@ public void testGuessRootCause() { assertEquals(ElasticsearchException.getExceptionName(rootCauses[0]), "index_not_found_exception"); assertEquals("no such index [foo]", rootCauses[0].getMessage()); ShardSearchFailure failure = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null), - new SearchShardTarget("node_1", new Index("foo", "_na_"), 1, null)); + new SearchShardTarget("node_1", new ShardId("foo", "_na_", 1), null, OriginalIndices.NONE)); ShardSearchFailure failure1 = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null), - new SearchShardTarget("node_1", new Index("foo", "_na_"), 2, null)); + new SearchShardTarget("node_1", new ShardId("foo", "_na_", 2), null, OriginalIndices.NONE)); SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed", new ShardSearchFailure[]{failure, failure1}); if (randomBoolean()) { @@ -136,11 +136,11 @@ public void testGuessRootCause() { { ShardSearchFailure failure = new ShardSearchFailure( new ParsingException(1, 2, "foobar", null), - new SearchShardTarget("node_1", new Index("foo", "_na_"), 1, null)); + new SearchShardTarget("node_1", new ShardId("foo", "_na_", 1), null, OriginalIndices.NONE)); ShardSearchFailure failure1 = new ShardSearchFailure(new QueryShardException(new Index("foo1", "_na_"), "foobar", null), - new SearchShardTarget("node_1", new Index("foo1", "_na_"), 1, null)); + new SearchShardTarget("node_1", new ShardId("foo1", "_na_", 1), null, OriginalIndices.NONE)); ShardSearchFailure failure2 = new ShardSearchFailure(new QueryShardException(new Index("foo1", "_na_"), "foobar", null), - new SearchShardTarget("node_1", new Index("foo1", "_na_"), 2, null)); + new SearchShardTarget("node_1", new ShardId("foo1", "_na_", 2), null, OriginalIndices.NONE)); SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed", new ShardSearchFailure[]{failure, failure1, failure2}); final ElasticsearchException[] rootCauses = ex.guessRootCauses(); @@ -187,9 +187,9 @@ public void testGuessRootCause() { public void testDeduplicate() throws IOException { { ShardSearchFailure failure = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null), - new SearchShardTarget("node_1", new Index("foo", "_na_"), 1, null)); + new SearchShardTarget("node_1", new ShardId("foo", "_na_", 1), null, OriginalIndices.NONE)); ShardSearchFailure failure1 = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null), - new SearchShardTarget("node_1", new Index("foo", "_na_"), 2, null)); + new SearchShardTarget("node_1", new ShardId("foo", "_na_", 2), null, OriginalIndices.NONE)); SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed", randomBoolean() ? failure1.getCause() : failure.getCause(), new ShardSearchFailure[]{failure, failure1}); XContentBuilder builder = XContentFactory.jsonBuilder(); @@ -203,11 +203,11 @@ public void testDeduplicate() throws IOException { } { ShardSearchFailure failure = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null), - new SearchShardTarget("node_1", new Index("foo", "_na_"), 1, null)); + new SearchShardTarget("node_1", new ShardId("foo", "_na_", 1), null, OriginalIndices.NONE)); ShardSearchFailure failure1 = new ShardSearchFailure(new QueryShardException(new Index("foo1", "_na_"), "foobar", null), - new SearchShardTarget("node_1", new Index("foo1", "_na_"), 1, null)); + new SearchShardTarget("node_1", new ShardId("foo1", "_na_", 1), null, OriginalIndices.NONE)); ShardSearchFailure failure2 = new ShardSearchFailure(new QueryShardException(new Index("foo1", "_na_"), "foobar", null), - new SearchShardTarget("node_1", new Index("foo1", "_na_"), 2, null)); + new SearchShardTarget("node_1", new ShardId("foo1", "_na_", 2), null, OriginalIndices.NONE)); SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed", new ShardSearchFailure[]{failure, failure1, failure2}); XContentBuilder builder = XContentFactory.jsonBuilder(); @@ -223,9 +223,9 @@ public void testDeduplicate() throws IOException { } { ShardSearchFailure failure = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null), - new SearchShardTarget("node_1", new Index("foo", "_na_"), 1, null)); + new SearchShardTarget("node_1", new ShardId("foo", "_na_", 1), null, OriginalIndices.NONE)); ShardSearchFailure failure1 = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null), - new SearchShardTarget("node_1", new Index("foo", "_na_"), 2, null)); + new SearchShardTarget("node_1", new ShardId("foo", "_na_", 2), null, OriginalIndices.NONE)); NullPointerException nullPointerException = new NullPointerException(); SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed", nullPointerException, new ShardSearchFailure[]{failure, failure1}); @@ -932,7 +932,7 @@ public static Tuple randomExceptions() { actual = new SearchPhaseExecutionException("search", "all shards failed", new ShardSearchFailure[]{ new ShardSearchFailure(new ParsingException(1, 2, "foobar", null), - new SearchShardTarget("node_1", new Index("foo", "_na_"), 1, null)) + new SearchShardTarget("node_1", new ShardId("foo", "_na_", 1), null, OriginalIndices.NONE)) }); expected = new ElasticsearchException("Elasticsearch exception [type=search_phase_execution_exception, " + "reason=all shards failed]"); diff --git a/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java index 97ce870d1bada..00354fd19dacc 100644 --- a/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java +++ b/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java @@ -24,6 +24,7 @@ import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.LockObtainFailedException; import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.RoutingMissingException; import org.elasticsearch.action.TimestampParsingException; import org.elasticsearch.action.search.SearchPhaseExecutionException; @@ -278,7 +279,7 @@ public void testQueryShardException() throws IOException { } public void testSearchException() throws IOException { - SearchShardTarget target = new SearchShardTarget("foo", new Index("bar", "_na_"), 1, null); + SearchShardTarget target = new SearchShardTarget("foo", new ShardId("bar", "_na_", 1), null, OriginalIndices.NONE); SearchException ex = serialize(new SearchException(target, "hello world")); assertEquals(target, ex.shard()); assertEquals(ex.getMessage(), "hello world"); diff --git a/server/src/test/java/org/elasticsearch/action/search/CountedCollectorTests.java b/server/src/test/java/org/elasticsearch/action/search/CountedCollectorTests.java index 951b5997508ca..a85f63abd0967 100644 --- a/server/src/test/java/org/elasticsearch/action/search/CountedCollectorTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/CountedCollectorTests.java @@ -18,8 +18,9 @@ */ package org.elasticsearch.action.search; +import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.common.util.concurrent.AtomicArray; -import org.elasticsearch.index.Index; +import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.dfs.DfsSearchResult; @@ -61,13 +62,13 @@ public void testCollect() throws InterruptedException { DfsSearchResult dfsSearchResult = new DfsSearchResult(shardID, null); dfsSearchResult.setShardIndex(shardID); dfsSearchResult.setSearchShardTarget(new SearchShardTarget("foo", - new Index("bar", "baz"), shardID, null)); + new ShardId("bar", "baz", shardID), null, OriginalIndices.NONE)); collector.onResult(dfsSearchResult);}); break; case 2: state.add(2); - executor.execute(() -> collector.onFailure(shardID, new SearchShardTarget("foo", new Index("bar", "baz"), - shardID, null), new RuntimeException("boom"))); + executor.execute(() -> collector.onFailure(shardID, new SearchShardTarget("foo", new ShardId("bar", "baz", shardID), + null, OriginalIndices.NONE), new RuntimeException("boom"))); break; default: fail("unknown state"); diff --git a/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java index c7a3137230405..8a8e28e15f20d 100644 --- a/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java @@ -24,10 +24,11 @@ import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TotalHits; import org.apache.lucene.store.MockDirectoryWrapper; +import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.concurrent.AtomicArray; -import org.elasticsearch.index.Index; +import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchShardTarget; @@ -53,8 +54,8 @@ private static DfsSearchResult newSearchResult(int shardIndex, long requestId, S public void testDfsWith2Shards() throws IOException { AtomicArray results = new AtomicArray<>(2); AtomicReference> responseRef = new AtomicReference<>(); - results.set(0, newSearchResult(0, 1, new SearchShardTarget("node1", new Index("test", "na"), 0, null))); - results.set(1, newSearchResult(1, 2, new SearchShardTarget("node2", new Index("test", "na"), 0, null))); + results.set(0, newSearchResult(0, 1, new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE))); + results.set(1, newSearchResult(1, 2, new SearchShardTarget("node2", new ShardId("test", "na", 0), null, OriginalIndices.NONE))); results.get(0).termsStatistics(new Term[0], new TermStatistics[0]); results.get(1).termsStatistics(new Term[0], new TermStatistics[0]); @@ -65,16 +66,16 @@ public void testDfsWith2Shards() throws IOException { public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest request, SearchTask task, SearchActionListener listener) { if (request.id() == 1) { - QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new Index("test", "na"), 0, - null)); + QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0), + null, OriginalIndices.NONE)); queryResult.topDocs(new TopDocsAndMaxScore( new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]); queryResult.size(2); // the size of the result set listener.onResponse(queryResult); } else if (request.id() == 2) { - QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node2", new Index("test", "na"), 0, - null)); + QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node2", new ShardId("test", "na", 0), + null, OriginalIndices.NONE)); queryResult.topDocs(new TopDocsAndMaxScore( new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[] {new ScoreDoc(84, 2.0F)}), 2.0F), new DocValueFormat[0]); @@ -113,8 +114,8 @@ public void run() throws IOException { public void testDfsWith1ShardFailed() throws IOException { AtomicArray results = new AtomicArray<>(2); AtomicReference> responseRef = new AtomicReference<>(); - results.set(0, newSearchResult(0, 1, new SearchShardTarget("node1", new Index("test", "na"), 0, null))); - results.set(1, newSearchResult(1, 2, new SearchShardTarget("node2", new Index("test", "na"), 0, null))); + results.set(0, newSearchResult(0, 1, new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE))); + results.set(1, newSearchResult(1, 2, new SearchShardTarget("node2", new ShardId("test", "na", 0), null, OriginalIndices.NONE))); results.get(0).termsStatistics(new Term[0], new TermStatistics[0]); results.get(1).termsStatistics(new Term[0], new TermStatistics[0]); @@ -125,8 +126,8 @@ public void testDfsWith1ShardFailed() throws IOException { public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest request, SearchTask task, SearchActionListener listener) { if (request.id() == 1) { - QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new Index("test", "na"), 0, - null)); + QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0), + null, OriginalIndices.NONE)); queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs( new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]); @@ -170,8 +171,8 @@ public void run() throws IOException { public void testFailPhaseOnException() throws IOException { AtomicArray results = new AtomicArray<>(2); AtomicReference> responseRef = new AtomicReference<>(); - results.set(0, newSearchResult(0, 1, new SearchShardTarget("node1", new Index("test", "na"), 0, null))); - results.set(1, newSearchResult(1, 2, new SearchShardTarget("node2", new Index("test", "na"), 0, null))); + results.set(0, newSearchResult(0, 1, new SearchShardTarget("node1", new ShardId("test", "na", 0), null, OriginalIndices.NONE))); + results.set(1, newSearchResult(1, 2, new SearchShardTarget("node2", new ShardId("test", "na", 0), null, OriginalIndices.NONE))); results.get(0).termsStatistics(new Term[0], new TermStatistics[0]); results.get(1).termsStatistics(new Term[0], new TermStatistics[0]); @@ -182,8 +183,8 @@ public void testFailPhaseOnException() throws IOException { public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest request, SearchTask task, SearchActionListener listener) { if (request.id() == 1) { - QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new Index("test", "na"), 0, - null)); + QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0), + null, OriginalIndices.NONE)); queryResult.topDocs(new TopDocsAndMaxScore( new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]); @@ -206,7 +207,7 @@ public void run() throws IOException { } }, mockSearchPhaseContext); assertEquals("dfs_query", phase.getName()); - expectThrows(UncheckedIOException.class, () -> phase.run()); + expectThrows(UncheckedIOException.class, phase::run); assertTrue(mockSearchPhaseContext.releasedSearchContexts.isEmpty()); // phase execution will clean up on the contexts } diff --git a/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java index b00c7f0bb152e..5614476c851b8 100644 --- a/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java @@ -22,9 +22,10 @@ import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TotalHits; import org.apache.lucene.store.MockDirectoryWrapper; +import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore; import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.index.Index; +import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; @@ -95,14 +96,15 @@ public void testFetchTwoDocument() throws IOException { controller.newSearchPhaseResults(mockSearchPhaseContext.getRequest(), 2); AtomicReference responseRef = new AtomicReference<>(); int resultSetSize = randomIntBetween(2, 10); - QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new Index("test", "na"), 0, null)); + QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0), + null, OriginalIndices.NONE)); queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]); queryResult.size(resultSetSize); // the size of the result set queryResult.setShardIndex(0); results.consumeResult(queryResult); - queryResult = new QuerySearchResult(321, new SearchShardTarget("node2", new Index("test", "na"), 1, null)); + queryResult = new QuerySearchResult(321, new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE)); queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[] {new ScoreDoc(84, 2.0F)}), 2.0F), new DocValueFormat[0]); queryResult.size(resultSetSize); @@ -153,14 +155,15 @@ public void testFailFetchOneDoc() throws IOException { controller.newSearchPhaseResults(mockSearchPhaseContext.getRequest(), 2); AtomicReference responseRef = new AtomicReference<>(); int resultSetSize = randomIntBetween(2, 10); - QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new Index("test", "na"), 0, null)); + QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0), + null, OriginalIndices.NONE)); queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]); queryResult.size(resultSetSize); // the size of the result set queryResult.setShardIndex(0); results.consumeResult(queryResult); - queryResult = new QuerySearchResult(321, new SearchShardTarget("node2", new Index("test", "na"), 1, null)); + queryResult = new QuerySearchResult(321, new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE)); queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[] {new ScoreDoc(84, 2.0F)}), 2.0F), new DocValueFormat[0]); queryResult.size(resultSetSize); @@ -215,7 +218,8 @@ public void testFetchDocsConcurrently() throws IOException, InterruptedException controller.newSearchPhaseResults(mockSearchPhaseContext.getRequest(), numHits); AtomicReference responseRef = new AtomicReference<>(); for (int i = 0; i < numHits; i++) { - QuerySearchResult queryResult = new QuerySearchResult(i, new SearchShardTarget("node1", new Index("test", "na"), 0, null)); + QuerySearchResult queryResult = new QuerySearchResult(i, new SearchShardTarget("node1", new ShardId("test", "na", 0), + null, OriginalIndices.NONE)); queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[] {new ScoreDoc(i+1, i)}), i), new DocValueFormat[0]); queryResult.size(resultSetSize); // the size of the result set @@ -272,14 +276,15 @@ public void testExceptionFailsPhase() throws IOException { controller.newSearchPhaseResults(mockSearchPhaseContext.getRequest(), 2); AtomicReference responseRef = new AtomicReference<>(); int resultSetSize = randomIntBetween(2, 10); - QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new Index("test", "na"), 0, null)); + QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0), + null, OriginalIndices.NONE)); queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]); queryResult.size(resultSetSize); // the size of the result set queryResult.setShardIndex(0); results.consumeResult(queryResult); - queryResult = new QuerySearchResult(321, new SearchShardTarget("node2", new Index("test", "na"), 1, null)); + queryResult = new QuerySearchResult(321, new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE)); queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[] {new ScoreDoc(84, 2.0F)}), 2.0F), new DocValueFormat[0]); queryResult.size(resultSetSize); @@ -329,14 +334,15 @@ public void testCleanupIrrelevantContexts() throws IOException { // contexts tha controller.newSearchPhaseResults(mockSearchPhaseContext.getRequest(), 2); AtomicReference responseRef = new AtomicReference<>(); int resultSetSize = 1; - QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new Index("test", "na"), 0, null)); + QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0), + null, OriginalIndices.NONE)); queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[] {new ScoreDoc(42, 1.0F)}), 2.0F), new DocValueFormat[0]); queryResult.size(resultSetSize); // the size of the result set queryResult.setShardIndex(0); results.consumeResult(queryResult); - queryResult = new QuerySearchResult(321, new SearchShardTarget("node2", new Index("test", "na"), 1, null)); + queryResult = new QuerySearchResult(321, new SearchShardTarget("node2", new ShardId("test", "na", 1), null, OriginalIndices.NONE)); queryResult.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[] {new ScoreDoc(84, 2.0F)}), 2.0F), new DocValueFormat[0]); queryResult.size(resultSetSize); diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java index e262147ef85d0..e9cde3f7aadea 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java @@ -35,7 +35,6 @@ import org.elasticsearch.common.text.Text; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.concurrent.AtomicArray; -import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchHit; @@ -321,7 +320,8 @@ public void testConsumer() { request.setBatchedReduceSize(bufferSize); InitialSearchPhase.ArraySearchPhaseResults consumer = searchPhaseController.newSearchPhaseResults(request, 3); assertEquals(0, reductions.size()); - QuerySearchResult result = new QuerySearchResult(0, new SearchShardTarget("node", new Index("a", "b"), 0, null)); + QuerySearchResult result = new QuerySearchResult(0, new SearchShardTarget("node", new ShardId("a", "b", 0), + null, OriginalIndices.NONE)); result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), Float.NaN), new DocValueFormat[0]); InternalAggregations aggs = new InternalAggregations(Collections.singletonList(new InternalMax("test", 1.0D, DocValueFormat.RAW, @@ -330,7 +330,7 @@ public void testConsumer() { result.setShardIndex(0); consumer.consumeResult(result); - result = new QuerySearchResult(1, new SearchShardTarget("node", new Index("a", "b"), 0, null)); + result = new QuerySearchResult(1, new SearchShardTarget("node", new ShardId("a", "b", 0), null, OriginalIndices.NONE)); result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), Float.NaN), new DocValueFormat[0]); aggs = new InternalAggregations(Collections.singletonList(new InternalMax("test", 3.0D, DocValueFormat.RAW, @@ -339,7 +339,7 @@ public void testConsumer() { result.setShardIndex(2); consumer.consumeResult(result); - result = new QuerySearchResult(1, new SearchShardTarget("node", new Index("a", "b"), 0, null)); + result = new QuerySearchResult(1, new SearchShardTarget("node", new ShardId("a", "b", 0), null, OriginalIndices.NONE)); result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), Float.NaN), new DocValueFormat[0]); aggs = new InternalAggregations(Collections.singletonList(new InternalMax("test", 2.0D, DocValueFormat.RAW, @@ -389,7 +389,8 @@ public void testConsumerConcurrently() throws InterruptedException { threads[i] = new Thread(() -> { int number = randomIntBetween(1, 1000); max.updateAndGet(prev -> Math.max(prev, number)); - QuerySearchResult result = new QuerySearchResult(id, new SearchShardTarget("node", new Index("a", "b"), id, null)); + QuerySearchResult result = new QuerySearchResult(id, new SearchShardTarget("node", new ShardId("a", "b", id), + null, OriginalIndices.NONE)); result.topDocs(new TopDocsAndMaxScore( new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[] {new ScoreDoc(0, number)}), number), new DocValueFormat[0]); @@ -432,7 +433,8 @@ public void testConsumerOnlyAggs() { for (int i = 0; i < expectedNumResults; i++) { int number = randomIntBetween(1, 1000); max.updateAndGet(prev -> Math.max(prev, number)); - QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new Index("a", "b"), i, null)); + QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new ShardId("a", "b", i), + null, OriginalIndices.NONE)); result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]), number), new DocValueFormat[0]); InternalAggregations aggs = new InternalAggregations(Collections.singletonList(new InternalMax("test", (double) number, @@ -469,7 +471,8 @@ public void testConsumerOnlyHits() { for (int i = 0; i < expectedNumResults; i++) { int number = randomIntBetween(1, 1000); max.updateAndGet(prev -> Math.max(prev, number)); - QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new Index("a", "b"), i, null)); + QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new ShardId("a", "b", i), + null, OriginalIndices.NONE)); result.topDocs(new TopDocsAndMaxScore(new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[] {new ScoreDoc(0, number)}), number), new DocValueFormat[0]); result.setShardIndex(i); @@ -536,7 +539,8 @@ public void testReduceTopNWithFromOffset() { searchPhaseController.newSearchPhaseResults(request, 4); int score = 100; for (int i = 0; i < 4; i++) { - QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new Index("a", "b"), i, null)); + QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new ShardId("a", "b", i), + null, OriginalIndices.NONE)); ScoreDoc[] docs = new ScoreDoc[3]; for (int j = 0; j < docs.length; j++) { docs[j] = new ScoreDoc(0, score--); @@ -577,7 +581,8 @@ public void testConsumerSortByField() { max.updateAndGet(prev -> Math.max(prev, number)); FieldDoc[] fieldDocs = {new FieldDoc(0, Float.NaN, new Object[]{number})}; TopDocs topDocs = new TopFieldDocs(new TotalHits(1, Relation.EQUAL_TO), fieldDocs, sortFields); - QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new Index("a", "b"), i, null)); + QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new ShardId("a", "b", i), + null, OriginalIndices.NONE)); result.topDocs(new TopDocsAndMaxScore(topDocs, Float.NaN), docValueFormats); result.setShardIndex(i); result.size(size); @@ -614,7 +619,8 @@ public void testConsumerFieldCollapsing() { Object[] values = {randomFrom(collapseValues)}; FieldDoc[] fieldDocs = {new FieldDoc(0, Float.NaN, values)}; TopDocs topDocs = new CollapseTopFieldDocs("field", new TotalHits(1, Relation.EQUAL_TO), fieldDocs, sortFields, values); - QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new Index("a", "b"), i, null)); + QuerySearchResult result = new QuerySearchResult(i, new SearchShardTarget("node", new ShardId("a", "b", i), + null, OriginalIndices.NONE)); result.topDocs(new TopDocsAndMaxScore(topDocs, Float.NaN), docValueFormats); result.setShardIndex(i); result.size(size); diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseExecutionExceptionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseExecutionExceptionTests.java index 9fbf3704fff21..f7a8f51d564fa 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseExecutionExceptionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseExecutionExceptionTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.action.search; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.TimestampParsingException; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; @@ -28,7 +29,6 @@ import org.elasticsearch.common.xcontent.XContent; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.IndexShardClosedException; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.InvalidIndexTemplateException; @@ -46,11 +46,11 @@ public void testToXContent() throws IOException { SearchPhaseExecutionException exception = new SearchPhaseExecutionException("test", "all shards failed", new ShardSearchFailure[]{ new ShardSearchFailure(new ParsingException(1, 2, "foobar", null), - new SearchShardTarget("node_1", new Index("foo", "_na_"), 0, null)), - new ShardSearchFailure(new IndexShardClosedException(new ShardId(new Index("foo", "_na_"), 1)), - new SearchShardTarget("node_2", new Index("foo", "_na_"), 1, null)), + new SearchShardTarget("node_1", new ShardId("foo", "_na_", 0), null, OriginalIndices.NONE)), + new ShardSearchFailure(new IndexShardClosedException(new ShardId("foo", "_na_", 1)), + new SearchShardTarget("node_2", new ShardId("foo", "_na_", 1), null, OriginalIndices.NONE)), new ShardSearchFailure(new ParsingException(5, 7, "foobar", null), - new SearchShardTarget("node_3", new Index("foo", "_na_"), 2, null)), + new SearchShardTarget("node_3", new ShardId("foo", "_na_", 2), null, OriginalIndices.NONE)), }); // Failures are grouped (by default) @@ -97,7 +97,8 @@ public void testToAndFromXContent() throws IOException { new TimestampParsingException("foo", null), new NullPointerException() ); - shardSearchFailures[i] = new ShardSearchFailure(cause, new SearchShardTarget("node_" + i, new Index("test", "_na_"), i, null)); + shardSearchFailures[i] = new ShardSearchFailure(cause, new SearchShardTarget("node_" + i, + new ShardId("test", "_na_", i), null, OriginalIndices.NONE)); } final String phase = randomFrom("query", "search", "other"); diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java index fbc3b1975def5..b902fba04c5e7 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchScrollAsyncActionTests.java @@ -20,11 +20,12 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AtomicArray; -import org.elasticsearch.index.Index; +import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.Scroll; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.internal.InternalScrollSearchRequest; @@ -71,7 +72,7 @@ protected void executeInitialPhase(Transport.Connection connection, InternalScro SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult = new SearchAsyncActionTests.TestSearchPhaseResult(internalRequest.id(), connection.getNode()); testSearchPhaseResult.setSearchShardTarget(new SearchShardTarget(connection.getNode().getId(), - new Index("test", "_na_"), 1, null)); + new ShardId("test", "_na_", 1), null, OriginalIndices.NONE)); searchActionListener.onResponse(testSearchPhaseResult); }).start(); } @@ -162,7 +163,7 @@ protected void executeInitialPhase(Transport.Connection connection, InternalScro SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult = new SearchAsyncActionTests.TestSearchPhaseResult(internalRequest.id(), connection.getNode()); testSearchPhaseResult.setSearchShardTarget(new SearchShardTarget(connection.getNode().getId(), - new Index("test", "_na_"), 1, null)); + new ShardId("test", "_na_", 1), null, OriginalIndices.NONE)); searchActionListener.onResponse(testSearchPhaseResult); }).start(); } @@ -235,7 +236,7 @@ protected void executeInitialPhase(Transport.Connection connection, InternalScro SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult = new SearchAsyncActionTests.TestSearchPhaseResult(internalRequest.id(), connection.getNode()); testSearchPhaseResult.setSearchShardTarget(new SearchShardTarget(connection.getNode().getId(), - new Index("test", "_na_"), 1, null)); + new ShardId("test", "_na_", 1), null, OriginalIndices.NONE)); searchActionListener.onResponse(testSearchPhaseResult); }).start(); } @@ -312,7 +313,7 @@ protected void executeInitialPhase(Transport.Connection connection, InternalScro SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult = new SearchAsyncActionTests.TestSearchPhaseResult(internalRequest.id(), connection.getNode()); testSearchPhaseResult.setSearchShardTarget(new SearchShardTarget(connection.getNode().getId(), - new Index("test", "_na_"), 1, null)); + new ShardId("test", "_na_", 1), null, OriginalIndices.NONE)); searchActionListener.onResponse(testSearchPhaseResult); } }).start(); diff --git a/server/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java b/server/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java index 29a7944f58792..3beb6fb8fdacf 100644 --- a/server/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java +++ b/server/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.common.ParsingException; @@ -32,7 +33,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.Index; +import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.FakeRestRequest; @@ -150,9 +151,9 @@ public void testConvert() throws IOException { RestRequest request = new FakeRestRequest(); RestChannel channel = new DetailedExceptionRestChannel(request); ShardSearchFailure failure = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null), - new SearchShardTarget("node_1", new Index("foo", "_na_"), 1, null)); + new SearchShardTarget("node_1", new ShardId("foo", "_na_", 1), null, OriginalIndices.NONE)); ShardSearchFailure failure1 = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null), - new SearchShardTarget("node_1", new Index("foo", "_na_"), 2, null)); + new SearchShardTarget("node_1", new ShardId("foo", "_na_", 2), null, OriginalIndices.NONE)); SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed", new ShardSearchFailure[] {failure, failure1}); BytesRestResponse response = new BytesRestResponse(channel, new RemoteTransportException("foo", ex)); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java index 5009594160ef7..63415d981e61d 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java @@ -20,6 +20,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; +import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.InputStreamStreamInput; import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; @@ -33,7 +34,7 @@ import org.elasticsearch.common.xcontent.XContentParseException; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.index.Index; +import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.SearchShardTarget; @@ -87,7 +88,8 @@ public int numberOfShards() { @Override public SearchShardTarget shardTarget() { - return new SearchShardTarget("no node, this is a unit test", new Index("no index, this is a unit test", "_na_"), 0, null); + return new SearchShardTarget("no node, this is a unit test", new ShardId("no index, this is a unit test", "_na_", 0), + null, OriginalIndices.NONE); } } diff --git a/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java b/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java index 1448590feb327..a9948b8b64856 100644 --- a/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java +++ b/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java @@ -21,13 +21,14 @@ import org.apache.lucene.search.Query; import org.elasticsearch.Version; +import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.TestSearchContext; @@ -45,7 +46,7 @@ public void testAssertNoInFlightContext() { @Override public SearchShardTarget shardTarget() { - return new SearchShardTarget("node", new Index("idx", "ignored"), 0, null); + return new SearchShardTarget("node", new ShardId("idx", "ignored", 0), null, OriginalIndices.NONE); } @Override diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java index 03751637a79a5..ab9fff7dbe587 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java @@ -8,6 +8,7 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.admin.indices.refresh.RefreshAction; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; @@ -181,7 +182,7 @@ void stopExecutor() { String id = String.valueOf(i); SearchHit hit = new SearchHit(1, id, new Text("watch"), Collections.emptyMap()); hit.version(1L); - hit.shard(new SearchShardTarget("nodeId", watchIndex, 0, "whatever")); + hit.shard(new SearchShardTarget("nodeId", new ShardId(watchIndex, 0), "whatever", OriginalIndices.NONE)); hits[i] = hit; boolean active = randomBoolean(); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/condition/CompareConditionSearchTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/condition/CompareConditionSearchTests.java index 5c525e10a1919..8824af4a8762a 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/condition/CompareConditionSearchTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/condition/CompareConditionSearchTests.java @@ -6,11 +6,12 @@ package org.elasticsearch.xpack.watcher.condition; import org.apache.lucene.search.TotalHits; +import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.index.Index; +import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.SearchShardTarget; @@ -77,7 +78,7 @@ public void testExecuteAccessHits() throws Exception { Clock.systemUTC()); SearchHit hit = new SearchHit(0, "1", new Text("type"), null); hit.score(1f); - hit.shard(new SearchShardTarget("a", new Index("a", "indexUUID"), 0, null)); + hit.shard(new SearchShardTarget("a", new ShardId("a", "indexUUID", 0), null, OriginalIndices.NONE)); InternalSearchResponse internalSearchResponse = new InternalSearchResponse( new SearchHits(new SearchHit[]{hit}, new TotalHits(1L, TotalHits.Relation.EQUAL_TO), 1f), diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java index 11656d838a076..e8e0e16b6ea4f 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.admin.indices.refresh.RefreshAction; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.bulk.BulkAction; @@ -216,7 +217,7 @@ public void testFindTriggeredWatchesGoodCase() { BytesArray source = new BytesArray("{}"); SearchHit hit = new SearchHit(0, "first_foo", new Text(TriggeredWatchStoreField.DOC_TYPE), null); hit.version(1L); - hit.shard(new SearchShardTarget("_node_id", index, 0, null)); + hit.shard(new SearchShardTarget("_node_id", new ShardId(index, 0), null, OriginalIndices.NONE)); hit.sourceRef(source); SearchHits hits = new SearchHits(new SearchHit[]{hit}, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0f); when(searchResponse1.getHits()).thenReturn(hits); @@ -230,7 +231,7 @@ public void testFindTriggeredWatchesGoodCase() { // First return a scroll response with a single hit and then with no hits hit = new SearchHit(0, "second_foo", new Text(TriggeredWatchStoreField.DOC_TYPE), null); hit.version(1L); - hit.shard(new SearchShardTarget("_node_id", index, 0, null)); + hit.shard(new SearchShardTarget("_node_id", new ShardId(index, 0), null, OriginalIndices.NONE)); hit.sourceRef(source); hits = new SearchHits(new SearchHit[]{hit}, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0f); SearchResponse searchResponse2 = new SearchResponse( From 7f1784e9f9a3fe370a398875c07f03cd34133671 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 29 Jan 2019 15:08:11 +0100 Subject: [PATCH 011/100] Remove Dead MockTransport Code (#34044) * All these methods are unused --- .../test/gateway/NoopGatewayAllocator.java | 50 ------------------- .../test/transport/MockTransportService.java | 47 ----------------- .../test/transport/StubbableTransport.java | 6 --- 3 files changed, 103 deletions(-) delete mode 100644 test/framework/src/main/java/org/elasticsearch/test/gateway/NoopGatewayAllocator.java diff --git a/test/framework/src/main/java/org/elasticsearch/test/gateway/NoopGatewayAllocator.java b/test/framework/src/main/java/org/elasticsearch/test/gateway/NoopGatewayAllocator.java deleted file mode 100644 index 9966bfb47fa47..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/test/gateway/NoopGatewayAllocator.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.test.gateway; - -import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.routing.allocation.FailedShard; -import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; -import org.elasticsearch.gateway.GatewayAllocator; - -import java.util.List; - -/** - * An allocator used for tests that doesn't do anything - */ -public class NoopGatewayAllocator extends GatewayAllocator { - - public static final NoopGatewayAllocator INSTANCE = new NoopGatewayAllocator(); - - @Override - public void applyStartedShards(RoutingAllocation allocation, List startedShards) { - // noop - } - - @Override - public void applyFailedShards(RoutingAllocation allocation, List failedShards) { - // noop - } - - @Override - public void allocateUnassigned(RoutingAllocation allocation) { - // noop - } -} diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java index 403ac96104a10..1ff1fa37fc120 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java @@ -237,13 +237,6 @@ public void addFailToSendNoConnectRule(TransportService transportService, final addFailToSendNoConnectRule(transportService, new HashSet<>(Arrays.asList(blockedActions))); } - /** - * Adds a rule that will cause matching operations to throw ConnectTransportExceptions - */ - public void addFailToSendNoConnectRule(TransportAddress transportAddress, final String... blockedActions) { - addFailToSendNoConnectRule(transportAddress, new HashSet<>(Arrays.asList(blockedActions))); - } - /** * Adds a rule that will cause matching operations to throw ConnectTransportExceptions */ @@ -413,16 +406,6 @@ public boolean addSendBehavior(TransportAddress transportAddress, StubbableTrans return transport().addSendBehavior(transportAddress, sendBehavior); } - /** - * Adds a send behavior that is the default send behavior. - * - * @return {@code true} if no default send behavior was registered - */ - public boolean addSendBehavior(StubbableTransport.SendRequestBehavior behavior) { - return transport().setDefaultSendBehavior(behavior); - } - - /** * Adds a new connect behavior that is used for creating connections with the given delegate service. * @@ -445,19 +428,6 @@ public boolean addConnectBehavior(TransportAddress transportAddress, StubbableTr return transport().addConnectBehavior(transportAddress, connectBehavior); } - /** - * Adds a new get connection behavior that is used for communication with the given delegate service. - * - * @return {@code true} if no other get connection behavior was registered for any of the addresses bound by delegate service. - */ - public boolean addGetConnectionBehavior(TransportService transportService, StubbableConnectionManager.GetConnectionBehavior behavior) { - boolean noRegistered = true; - for (TransportAddress transportAddress : extractTransportAddresses(transportService)) { - noRegistered &= addGetConnectionBehavior(transportAddress, behavior); - } - return noRegistered; - } - /** * Adds a get connection behavior that is used for communication with the given delegate address. * @@ -476,19 +446,6 @@ public boolean addGetConnectionBehavior(StubbableConnectionManager.GetConnection return connectionManager().setDefaultGetConnectionBehavior(behavior); } - /** - * Adds a node connected behavior that is used for the given delegate service. - * - * @return {@code true} if no other node connected behavior was registered for any of the addresses bound by delegate service. - */ - public boolean addNodeConnectedBehavior(TransportService transportService, StubbableConnectionManager.NodeConnectedBehavior behavior) { - boolean noRegistered = true; - for (TransportAddress transportAddress : extractTransportAddresses(transportService)) { - noRegistered &= addNodeConnectedBehavior(transportAddress, behavior); - } - return noRegistered; - } - /** * Adds a node connected behavior that is used for the given delegate address. * @@ -538,10 +495,6 @@ public void addTracer(Tracer tracer) { activeTracers.add(tracer); } - public boolean removeTracer(Tracer tracer) { - return activeTracers.remove(tracer); - } - public void clearTracers() { activeTracers.clear(); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java index 0014e1225c595..4f0df85fd50c8 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java @@ -55,12 +55,6 @@ public StubbableTransport(Transport transport) { this.delegate = transport; } - boolean setDefaultSendBehavior(SendRequestBehavior sendBehavior) { - SendRequestBehavior prior = defaultSendRequest; - defaultSendRequest = sendBehavior; - return prior == null; - } - public boolean setDefaultConnectBehavior(OpenConnectionBehavior openConnectionBehavior) { OpenConnectionBehavior prior = this.defaultConnectBehavior; this.defaultConnectBehavior = openConnectionBehavior; From 5d1964bcbf5c4dcdf9ea8e567a1b4211f47aff33 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Tue, 29 Jan 2019 15:09:40 +0100 Subject: [PATCH 012/100] Ignore shard started requests when primary term does not match (#37899) This commit changes the StartedShardEntry so that it also contains the primary term of the shard to start. This way the master node can also checks that the primary term from the start request is equal to the current shard's primary term in the cluster state, and it can ignore any shard started request that would concerns a previous instance of the shard that would have been allocated to the same node. Such situation are likely to happen with frozen (or restored) indices and the replication of closed indices, because with replicated closed indices the shards will be initialized again after the index is closed and can potentially be re initialized again if the index is reopened as a frozen index. In such cases the lifecycle of the shards would be something like: * shard is STARTED * index is closed * shards is INITIALIZING (index state is CLOSED, primary term is X) * index is reopened * shards are INITIALIZING again (index state is OPENED, potentially frozen, primary term is X+1) Adding the primary term to the shard started request will allow to discard potential StartedShardEntry requests received by the master node if the request concerns the shard with primary term X because it has been moved/reinitialized in the meanwhile under the primary term X+1. Relates to #33888 --- .../action/shard/ShardStateAction.java | 49 ++++-- .../cluster/IndicesClusterStateService.java | 25 ++- ...dStartedClusterStateTaskExecutorTests.java | 166 ++++++++++++------ .../action/shard/ShardStateActionTests.java | 61 ++++++- ...actIndicesClusterStateServiceTestCase.java | 9 +- .../indices/cluster/ClusterStateChanges.java | 16 +- ...ClusterStateServiceRandomUpdatesTests.java | 4 +- 7 files changed, 248 insertions(+), 82 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java b/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java index e5c46cbb0ee08..4419d921a3b4a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java +++ b/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java @@ -494,12 +494,20 @@ public int hashCode() { } } - public void shardStarted(final ShardRouting shardRouting, final String message, Listener listener) { - shardStarted(shardRouting, message, listener, clusterService.state()); + public void shardStarted(final ShardRouting shardRouting, + final long primaryTerm, + final String message, + final Listener listener) { + shardStarted(shardRouting, primaryTerm, message, listener, clusterService.state()); } - public void shardStarted(final ShardRouting shardRouting, final String message, Listener listener, ClusterState currentState) { - StartedShardEntry shardEntry = new StartedShardEntry(shardRouting.shardId(), shardRouting.allocationId().getId(), message); - sendShardAction(SHARD_STARTED_ACTION_NAME, currentState, shardEntry, listener); + + public void shardStarted(final ShardRouting shardRouting, + final long primaryTerm, + final String message, + final Listener listener, + final ClusterState currentState) { + StartedShardEntry entry = new StartedShardEntry(shardRouting.shardId(), shardRouting.allocationId().getId(), primaryTerm, message); + sendShardAction(SHARD_STARTED_ACTION_NAME, currentState, entry, listener); } private static class ShardStartedTransportHandler implements TransportRequestHandler { @@ -544,7 +552,7 @@ public ClusterTasksResult execute(ClusterState currentState, List shardRoutingsToBeApplied = new ArrayList<>(tasks.size()); Set seenShardRoutings = new HashSet<>(); // to prevent duplicates for (StartedShardEntry task : tasks) { - ShardRouting matched = currentState.getRoutingTable().getByAllocationId(task.shardId, task.allocationId); + final ShardRouting matched = currentState.getRoutingTable().getByAllocationId(task.shardId, task.allocationId); if (matched == null) { // tasks that correspond to non-existent shards are marked as successful. The reason is that we resend shard started // events on every cluster state publishing that does not contain the shard as started yet. This means that old stale @@ -553,6 +561,19 @@ public ClusterTasksResult execute(ClusterState currentState, logger.debug("{} ignoring shard started task [{}] (shard does not exist anymore)", task.shardId, task); builder.success(task); } else { + if (matched.primary() && task.primaryTerm > 0) { + final IndexMetaData indexMetaData = currentState.metaData().index(task.shardId.getIndex()); + assert indexMetaData != null; + final long currentPrimaryTerm = indexMetaData.primaryTerm(task.shardId.id()); + if (currentPrimaryTerm != task.primaryTerm) { + assert currentPrimaryTerm > task.primaryTerm : "received a primary term with a higher term than in the " + + "current cluster state (received [" + task.primaryTerm + "] but current is [" + currentPrimaryTerm + "])"; + logger.debug("{} ignoring shard started task [{}] (primary term {} does not match current term {})", + task.shardId, task, task.primaryTerm, currentPrimaryTerm); + builder.success(task); + continue; + } + } if (matched.initializing() == false) { assert matched.active() : "expected active shard routing for task " + task + " but found " + matched; // same as above, this might have been a stale in-flight request, so we just ignore. @@ -597,6 +618,7 @@ public void onFailure(String source, Exception e) { public static class StartedShardEntry extends TransportRequest { final ShardId shardId; final String allocationId; + final long primaryTerm; final String message; StartedShardEntry(StreamInput in) throws IOException { @@ -604,8 +626,12 @@ public static class StartedShardEntry extends TransportRequest { shardId = ShardId.readShardId(in); allocationId = in.readString(); if (in.getVersion().before(Version.V_6_3_0)) { - final long primaryTerm = in.readVLong(); + primaryTerm = in.readVLong(); assert primaryTerm == UNASSIGNED_PRIMARY_TERM : "shard is only started by itself: primary term [" + primaryTerm + "]"; + } else if (in.getVersion().onOrAfter(Version.V_7_0_0)) { // TODO update version to 6.7.0 after backport + primaryTerm = in.readVLong(); + } else { + primaryTerm = UNASSIGNED_PRIMARY_TERM; } this.message = in.readString(); if (in.getVersion().before(Version.V_6_3_0)) { @@ -614,9 +640,10 @@ public static class StartedShardEntry extends TransportRequest { } } - public StartedShardEntry(ShardId shardId, String allocationId, String message) { + public StartedShardEntry(final ShardId shardId, final String allocationId, final long primaryTerm, final String message) { this.shardId = shardId; this.allocationId = allocationId; + this.primaryTerm = primaryTerm; this.message = message; } @@ -627,6 +654,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(allocationId); if (out.getVersion().before(Version.V_6_3_0)) { out.writeVLong(0L); + } else if (out.getVersion().onOrAfter(Version.V_7_0_0)) { // TODO update version to 6.7.0 after backport + out.writeVLong(primaryTerm); } out.writeString(message); if (out.getVersion().before(Version.V_6_3_0)) { @@ -636,8 +665,8 @@ public void writeTo(StreamOutput out) throws IOException { @Override public String toString() { - return String.format(Locale.ROOT, "StartedShardEntry{shardId [%s], allocationId [%s], message [%s]}", - shardId, allocationId, message); + return String.format(Locale.ROOT, "StartedShardEntry{shardId [%s], allocationId [%s], primary term [%d], message [%s]}", + shardId, allocationId, primaryTerm, message); } } diff --git a/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java index 80ac05ece8274..5955a749fea34 100644 --- a/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java +++ b/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java @@ -575,13 +575,14 @@ private void createShard(DiscoveryNodes nodes, RoutingTable routingTable, ShardR } try { - logger.debug("{} creating shard", shardRouting.shardId()); + final long primaryTerm = state.metaData().index(shardRouting.index()).primaryTerm(shardRouting.id()); + logger.debug("{} creating shard with primary term [{}]", shardRouting.shardId(), primaryTerm); RecoveryState recoveryState = new RecoveryState(shardRouting, nodes.getLocalNode(), sourceNode); indicesService.createShard( shardRouting, recoveryState, recoveryTargetService, - new RecoveryListener(shardRouting), + new RecoveryListener(shardRouting, primaryTerm), repositoriesService, failedShardHandler, globalCheckpointSyncer, @@ -598,9 +599,10 @@ private void updateShard(DiscoveryNodes nodes, ShardRouting shardRouting, Shard "local shard has a different allocation id but wasn't cleaning by removeShards. " + "cluster state: " + shardRouting + " local: " + currentRoutingEntry; + final long primaryTerm; try { final IndexMetaData indexMetaData = clusterState.metaData().index(shard.shardId().getIndex()); - final long primaryTerm = indexMetaData.primaryTerm(shard.shardId().id()); + primaryTerm = indexMetaData.primaryTerm(shard.shardId().id()); final Set inSyncIds = indexMetaData.inSyncAllocationIds(shard.shardId().id()); final IndexShardRoutingTable indexShardRoutingTable = routingTable.shardRoutingTable(shardRouting.shardId()); final Set pre60AllocationIds = indexShardRoutingTable.assignedShards() @@ -633,7 +635,7 @@ private void updateShard(DiscoveryNodes nodes, ShardRouting shardRouting, Shard shardRouting.shardId(), state, nodes.getMasterNode()); } if (nodes.getMasterNode() != null) { - shardStateAction.shardStarted(shardRouting, "master " + nodes.getMasterNode() + + shardStateAction.shardStarted(shardRouting, primaryTerm, "master " + nodes.getMasterNode() + " marked shard as initializing, but shard state is [" + state + "], mark shard as started", SHARD_STATE_ACTION_LISTENER, clusterState); } @@ -673,15 +675,24 @@ private static DiscoveryNode findSourceNodeForPeerRecovery(Logger logger, Routin private class RecoveryListener implements PeerRecoveryTargetService.RecoveryListener { + /** + * ShardRouting with which the shard was created + */ private final ShardRouting shardRouting; - private RecoveryListener(ShardRouting shardRouting) { + /** + * Primary term with which the shard was created + */ + private final long primaryTerm; + + private RecoveryListener(final ShardRouting shardRouting, final long primaryTerm) { this.shardRouting = shardRouting; + this.primaryTerm = primaryTerm; } @Override - public void onRecoveryDone(RecoveryState state) { - shardStateAction.shardStarted(shardRouting, "after " + state.getRecoverySource(), SHARD_STATE_ACTION_LISTENER); + public void onRecoveryDone(final RecoveryState state) { + shardStateAction.shardStarted(shardRouting, primaryTerm, "after " + state.getRecoverySource(), SHARD_STATE_ACTION_LISTENER); } @Override diff --git a/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStartedClusterStateTaskExecutorTests.java b/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStartedClusterStateTaskExecutorTests.java index 1d3a523cdc94f..20b7548004f4a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStartedClusterStateTaskExecutorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStartedClusterStateTaskExecutorTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.cluster.ESAllocationTestCase; import org.elasticsearch.cluster.action.shard.ShardStateAction.StartedShardEntry; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; @@ -34,7 +35,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -64,19 +64,19 @@ public void setUp() throws Exception { public void testEmptyTaskListProducesSameClusterState() throws Exception { final ClusterState clusterState = stateWithNoShard(); - assertTasksExecution(clusterState, Collections.emptyList(), result -> assertSame(clusterState, result.resultingState)); + final ClusterStateTaskExecutor.ClusterTasksResult result = executeTasks(clusterState, Collections.emptyList()); + assertSame(clusterState, result.resultingState); } public void testNonExistentIndexMarkedAsSuccessful() throws Exception { final ClusterState clusterState = stateWithNoShard(); - final StartedShardEntry entry = new StartedShardEntry(new ShardId("test", "_na", 0), "aId", "test"); - assertTasksExecution(clusterState, singletonList(entry), - result -> { - assertSame(clusterState, result.resultingState); - assertThat(result.executionResults.size(), equalTo(1)); - assertThat(result.executionResults.containsKey(entry), is(true)); - assertThat(((ClusterStateTaskExecutor.TaskResult) result.executionResults.get(entry)).isSuccess(), is(true)); - }); + final StartedShardEntry entry = new StartedShardEntry(new ShardId("test", "_na", 0), "aId", randomNonNegativeLong(), "test"); + + final ClusterStateTaskExecutor.ClusterTasksResult result = executeTasks(clusterState, singletonList(entry)); + assertSame(clusterState, result.resultingState); + assertThat(result.executionResults.size(), equalTo(1)); + assertThat(result.executionResults.containsKey(entry), is(true)); + assertThat(((ClusterStateTaskExecutor.TaskResult) result.executionResults.get(entry)).isSuccess(), is(true)); } public void testNonExistentShardsAreMarkedAsSuccessful() throws Exception { @@ -87,20 +87,19 @@ public void testNonExistentShardsAreMarkedAsSuccessful() throws Exception { final List tasks = Stream.concat( // Existent shard id but different allocation id IntStream.range(0, randomIntBetween(1, 5)) - .mapToObj(i -> new StartedShardEntry(new ShardId(indexMetaData.getIndex(), 0), String.valueOf(i), "allocation id")), + .mapToObj(i -> new StartedShardEntry(new ShardId(indexMetaData.getIndex(), 0), String.valueOf(i), 0L, "allocation id")), // Non existent shard id IntStream.range(1, randomIntBetween(2, 5)) - .mapToObj(i -> new StartedShardEntry(new ShardId(indexMetaData.getIndex(), i), String.valueOf(i), "shard id")) + .mapToObj(i -> new StartedShardEntry(new ShardId(indexMetaData.getIndex(), i), String.valueOf(i), 0L, "shard id")) ).collect(Collectors.toList()); - assertTasksExecution(clusterState, tasks, result -> { - assertSame(clusterState, result.resultingState); - assertThat(result.executionResults.size(), equalTo(tasks.size())); - tasks.forEach(task -> { - assertThat(result.executionResults.containsKey(task), is(true)); - assertThat(((ClusterStateTaskExecutor.TaskResult) result.executionResults.get(task)).isSuccess(), is(true)); - }); + final ClusterStateTaskExecutor.ClusterTasksResult result = executeTasks(clusterState, tasks); + assertSame(clusterState, result.resultingState); + assertThat(result.executionResults.size(), equalTo(tasks.size())); + tasks.forEach(task -> { + assertThat(result.executionResults.containsKey(task), is(true)); + assertThat(((ClusterStateTaskExecutor.TaskResult) result.executionResults.get(task)).isSuccess(), is(true)); }); } @@ -119,16 +118,16 @@ public void testNonInitializingShardAreMarkedAsSuccessful() throws Exception { } else { allocationId = shardRoutingTable.replicaShards().iterator().next().allocationId().getId(); } - return new StartedShardEntry(shardId, allocationId, "test"); + final long primaryTerm = indexMetaData.primaryTerm(shardId.id()); + return new StartedShardEntry(shardId, allocationId, primaryTerm, "test"); }).collect(Collectors.toList()); - assertTasksExecution(clusterState, tasks, result -> { - assertSame(clusterState, result.resultingState); - assertThat(result.executionResults.size(), equalTo(tasks.size())); - tasks.forEach(task -> { - assertThat(result.executionResults.containsKey(task), is(true)); - assertThat(((ClusterStateTaskExecutor.TaskResult) result.executionResults.get(task)).isSuccess(), is(true)); - }); + final ClusterStateTaskExecutor.ClusterTasksResult result = executeTasks(clusterState, tasks); + assertSame(clusterState, result.resultingState); + assertThat(result.executionResults.size(), equalTo(tasks.size())); + tasks.forEach(task -> { + assertThat(result.executionResults.containsKey(task), is(true)); + assertThat(((ClusterStateTaskExecutor.TaskResult) result.executionResults.get(task)).isSuccess(), is(true)); }); } @@ -138,26 +137,26 @@ public void testStartedShards() throws Exception { final IndexMetaData indexMetaData = clusterState.metaData().index(indexName); final ShardId shardId = new ShardId(indexMetaData.getIndex(), 0); + final long primaryTerm = indexMetaData.primaryTerm(shardId.id()); final ShardRouting primaryShard = clusterState.routingTable().shardRoutingTable(shardId).primaryShard(); final String primaryAllocationId = primaryShard.allocationId().getId(); final List tasks = new ArrayList<>(); - tasks.add(new StartedShardEntry(shardId, primaryAllocationId, "test")); + tasks.add(new StartedShardEntry(shardId, primaryAllocationId, primaryTerm, "test")); if (randomBoolean()) { final ShardRouting replicaShard = clusterState.routingTable().shardRoutingTable(shardId).replicaShards().iterator().next(); final String replicaAllocationId = replicaShard.allocationId().getId(); - tasks.add(new StartedShardEntry(shardId, replicaAllocationId, "test")); + tasks.add(new StartedShardEntry(shardId, replicaAllocationId, primaryTerm, "test")); } - assertTasksExecution(clusterState, tasks, result -> { - assertNotSame(clusterState, result.resultingState); - assertThat(result.executionResults.size(), equalTo(tasks.size())); - tasks.forEach(task -> { - assertThat(result.executionResults.containsKey(task), is(true)); - assertThat(((ClusterStateTaskExecutor.TaskResult) result.executionResults.get(task)).isSuccess(), is(true)); - - final IndexShardRoutingTable shardRoutingTable = result.resultingState.routingTable().shardRoutingTable(task.shardId); - assertThat(shardRoutingTable.getByAllocationId(task.allocationId).state(), is(ShardRoutingState.STARTED)); - }); + final ClusterStateTaskExecutor.ClusterTasksResult result = executeTasks(clusterState, tasks); + assertNotSame(clusterState, result.resultingState); + assertThat(result.executionResults.size(), equalTo(tasks.size())); + tasks.forEach(task -> { + assertThat(result.executionResults.containsKey(task), is(true)); + assertThat(((ClusterStateTaskExecutor.TaskResult) result.executionResults.get(task)).isSuccess(), is(true)); + + final IndexShardRoutingTable shardRoutingTable = result.resultingState.routingTable().shardRoutingTable(task.shardId); + assertThat(shardRoutingTable.getByAllocationId(task.allocationId).state(), is(ShardRoutingState.STARTED)); }); } @@ -169,29 +168,88 @@ public void testDuplicateStartsAreOkay() throws Exception { final ShardId shardId = new ShardId(indexMetaData.getIndex(), 0); final ShardRouting shardRouting = clusterState.routingTable().shardRoutingTable(shardId).primaryShard(); final String allocationId = shardRouting.allocationId().getId(); + final long primaryTerm = indexMetaData.primaryTerm(shardId.id()); final List tasks = IntStream.range(0, randomIntBetween(2, 10)) - .mapToObj(i -> new StartedShardEntry(shardId, allocationId, "test")) + .mapToObj(i -> new StartedShardEntry(shardId, allocationId, primaryTerm, "test")) .collect(Collectors.toList()); - assertTasksExecution(clusterState, tasks, result -> { - assertNotSame(clusterState, result.resultingState); - assertThat(result.executionResults.size(), equalTo(tasks.size())); - tasks.forEach(task -> { - assertThat(result.executionResults.containsKey(task), is(true)); - assertThat(((ClusterStateTaskExecutor.TaskResult) result.executionResults.get(task)).isSuccess(), is(true)); - - final IndexShardRoutingTable shardRoutingTable = result.resultingState.routingTable().shardRoutingTable(task.shardId); - assertThat(shardRoutingTable.getByAllocationId(task.allocationId).state(), is(ShardRoutingState.STARTED)); - }); + final ClusterStateTaskExecutor.ClusterTasksResult result = executeTasks(clusterState, tasks); + assertNotSame(clusterState, result.resultingState); + assertThat(result.executionResults.size(), equalTo(tasks.size())); + tasks.forEach(task -> { + assertThat(result.executionResults.containsKey(task), is(true)); + assertThat(((ClusterStateTaskExecutor.TaskResult) result.executionResults.get(task)).isSuccess(), is(true)); + + final IndexShardRoutingTable shardRoutingTable = result.resultingState.routingTable().shardRoutingTable(task.shardId); + assertThat(shardRoutingTable.getByAllocationId(task.allocationId).state(), is(ShardRoutingState.STARTED)); }); } - private void assertTasksExecution(final ClusterState state, - final List tasks, - final Consumer consumer) throws Exception { + public void testPrimaryTermsMismatch() throws Exception { + final String indexName = "test"; + final int shard = 0; + final int primaryTerm = 2 + randomInt(200); + + ClusterState clusterState = state(indexName, randomBoolean(), ShardRoutingState.INITIALIZING, ShardRoutingState.INITIALIZING); + clusterState = ClusterState.builder(clusterState) + .metaData(MetaData.builder(clusterState.metaData()) + .put(IndexMetaData.builder(clusterState.metaData().index(indexName)) + .primaryTerm(shard, primaryTerm) + .build(), true) + .build()) + .build(); + final ShardId shardId = new ShardId(clusterState.metaData().index(indexName).getIndex(), shard); + final String primaryAllocationId = clusterState.routingTable().shardRoutingTable(shardId).primaryShard().allocationId().getId(); + { + final StartedShardEntry task = + new StartedShardEntry(shardId, primaryAllocationId, primaryTerm - 1, "primary terms does not match on primary"); + + final ClusterStateTaskExecutor.ClusterTasksResult result = executeTasks(clusterState, singletonList(task)); + assertSame(clusterState, result.resultingState); + assertThat(result.executionResults.size(), equalTo(1)); + assertThat(result.executionResults.containsKey(task), is(true)); + assertThat(((ClusterStateTaskExecutor.TaskResult) result.executionResults.get(task)).isSuccess(), is(true)); + IndexShardRoutingTable shardRoutingTable = result.resultingState.routingTable().shardRoutingTable(task.shardId); + assertThat(shardRoutingTable.getByAllocationId(task.allocationId).state(), is(ShardRoutingState.INITIALIZING)); + assertSame(clusterState, result.resultingState); + } + { + final StartedShardEntry task = + new StartedShardEntry(shardId, primaryAllocationId, primaryTerm, "primary terms match on primary"); + + final ClusterStateTaskExecutor.ClusterTasksResult result = executeTasks(clusterState, singletonList(task)); + assertNotSame(clusterState, result.resultingState); + assertThat(result.executionResults.size(), equalTo(1)); + assertThat(result.executionResults.containsKey(task), is(true)); + assertThat(((ClusterStateTaskExecutor.TaskResult) result.executionResults.get(task)).isSuccess(), is(true)); + IndexShardRoutingTable shardRoutingTable = result.resultingState.routingTable().shardRoutingTable(task.shardId); + assertThat(shardRoutingTable.getByAllocationId(task.allocationId).state(), is(ShardRoutingState.STARTED)); + assertNotSame(clusterState, result.resultingState); + clusterState = result.resultingState; + } + { + final long replicaPrimaryTerm = randomBoolean() ? primaryTerm : primaryTerm - 1; + final String replicaAllocationId = clusterState.routingTable().shardRoutingTable(shardId).replicaShards().iterator().next() + .allocationId().getId(); + + final StartedShardEntry task = new StartedShardEntry(shardId, replicaAllocationId, replicaPrimaryTerm, "test on replica"); + + final ClusterStateTaskExecutor.ClusterTasksResult result = executeTasks(clusterState, singletonList(task)); + assertNotSame(clusterState, result.resultingState); + assertThat(result.executionResults.size(), equalTo(1)); + assertThat(result.executionResults.containsKey(task), is(true)); + assertThat(((ClusterStateTaskExecutor.TaskResult) result.executionResults.get(task)).isSuccess(), is(true)); + IndexShardRoutingTable shardRoutingTable = result.resultingState.routingTable().shardRoutingTable(task.shardId); + assertThat(shardRoutingTable.getByAllocationId(task.allocationId).state(), is(ShardRoutingState.STARTED)); + assertNotSame(clusterState, result.resultingState); + } + } + + private ClusterStateTaskExecutor.ClusterTasksResult executeTasks(final ClusterState state, + final List tasks) throws Exception { final ClusterStateTaskExecutor.ClusterTasksResult result = executor.execute(state, tasks); assertThat(result, notNullValue()); - consumer.accept(result); + return result; } } diff --git a/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java b/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java index e94a974ae7a89..a800c0c79929c 100644 --- a/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java @@ -72,12 +72,14 @@ import static org.elasticsearch.test.ClusterServiceUtils.createClusterService; import static org.elasticsearch.test.ClusterServiceUtils.setState; +import static org.elasticsearch.test.VersionUtils.randomCompatibleVersion; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.sameInstance; import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; public class ShardStateActionTests extends ESTestCase { @@ -420,8 +422,9 @@ public void testShardStarted() throws InterruptedException { setState(clusterService, ClusterStateCreationUtils.stateWithActivePrimary(index, true, randomInt(5))); final ShardRouting shardRouting = getRandomShardRouting(index); + final long primaryTerm = clusterService.state().metaData().index(shardRouting.index()).primaryTerm(shardRouting.id()); final TestListener listener = new TestListener(); - shardStateAction.shardStarted(shardRouting, "testShardStarted", listener); + shardStateAction.shardStarted(shardRouting, primaryTerm, "testShardStarted", listener); final CapturingTransport.CapturedRequest[] capturedRequests = transport.getCapturedRequestsAndClear(); assertThat(capturedRequests[0].request, instanceOf(ShardStateAction.StartedShardEntry.class)); @@ -429,6 +432,7 @@ public void testShardStarted() throws InterruptedException { ShardStateAction.StartedShardEntry entry = (ShardStateAction.StartedShardEntry) capturedRequests[0].request; assertThat(entry.shardId, equalTo(shardRouting.shardId())); assertThat(entry.allocationId, equalTo(shardRouting.allocationId().getId())); + assertThat(entry.primaryTerm, equalTo(primaryTerm)); transport.handleResponse(capturedRequests[0].requestId, TransportResponse.Empty.INSTANCE); listener.await(); @@ -481,7 +485,7 @@ public void testShardEntryBWCSerialize() throws Exception { final ShardId shardId = new ShardId(randomRealisticUnicodeOfLengthBetween(10, 100), UUID.randomUUID().toString(), between(0, 1000)); final String allocationId = randomRealisticUnicodeOfCodepointLengthBetween(10, 100); final String reason = randomRealisticUnicodeOfCodepointLengthBetween(10, 100); - try (StreamInput in = serialize(new StartedShardEntry(shardId, allocationId, reason), bwcVersion).streamInput()) { + try (StreamInput in = serialize(new StartedShardEntry(shardId, allocationId, 0L, reason), bwcVersion).streamInput()) { in.setVersion(bwcVersion); final FailedShardEntry failedShardEntry = new FailedShardEntry(in); assertThat(failedShardEntry.shardId, equalTo(shardId)); @@ -490,8 +494,7 @@ public void testShardEntryBWCSerialize() throws Exception { assertThat(failedShardEntry.failure, nullValue()); assertThat(failedShardEntry.markAsStale, equalTo(true)); } - try (StreamInput in = serialize(new FailedShardEntry(shardId, allocationId, 0L, - reason, null, false), bwcVersion).streamInput()) { + try (StreamInput in = serialize(new FailedShardEntry(shardId, allocationId, 0L, reason, null, false), bwcVersion).streamInput()) { in.setVersion(bwcVersion); final StartedShardEntry startedShardEntry = new StartedShardEntry(in); assertThat(startedShardEntry.shardId, equalTo(shardId)); @@ -500,6 +503,56 @@ public void testShardEntryBWCSerialize() throws Exception { } } + public void testFailedShardEntrySerialization() throws Exception { + final ShardId shardId = new ShardId(randomRealisticUnicodeOfLengthBetween(10, 100), UUID.randomUUID().toString(), between(0, 1000)); + final String allocationId = randomRealisticUnicodeOfCodepointLengthBetween(10, 100); + final long primaryTerm = randomIntBetween(0, 100); + final String message = randomRealisticUnicodeOfCodepointLengthBetween(10, 100); + final Exception failure = randomBoolean() ? null : getSimulatedFailure(); + final boolean markAsStale = randomBoolean(); + + final Version version = randomFrom(randomCompatibleVersion(random(), Version.CURRENT)); + final FailedShardEntry failedShardEntry = new FailedShardEntry(shardId, allocationId, primaryTerm, message, failure, markAsStale); + try (StreamInput in = serialize(failedShardEntry, version).streamInput()) { + in.setVersion(version); + final FailedShardEntry deserialized = new FailedShardEntry(in); + assertThat(deserialized.shardId, equalTo(shardId)); + assertThat(deserialized.allocationId, equalTo(allocationId)); + assertThat(deserialized.primaryTerm, equalTo(primaryTerm)); + assertThat(deserialized.message, equalTo(message)); + if (failure != null) { + assertThat(deserialized.failure, notNullValue()); + assertThat(deserialized.failure.getClass(), equalTo(failure.getClass())); + assertThat(deserialized.failure.getMessage(), equalTo(failure.getMessage())); + } else { + assertThat(deserialized.failure, nullValue()); + } + assertThat(deserialized.markAsStale, equalTo(markAsStale)); + assertEquals(failedShardEntry, deserialized); + } + } + + public void testStartedShardEntrySerialization() throws Exception { + final ShardId shardId = new ShardId(randomRealisticUnicodeOfLengthBetween(10, 100), UUID.randomUUID().toString(), between(0, 1000)); + final String allocationId = randomRealisticUnicodeOfCodepointLengthBetween(10, 100); + final long primaryTerm = randomIntBetween(0, 100); + final String message = randomRealisticUnicodeOfCodepointLengthBetween(10, 100); + + final Version version = randomFrom(randomCompatibleVersion(random(), Version.CURRENT)); + try (StreamInput in = serialize(new StartedShardEntry(shardId, allocationId, primaryTerm, message), version).streamInput()) { + in.setVersion(version); + final StartedShardEntry deserialized = new StartedShardEntry(in); + assertThat(deserialized.shardId, equalTo(shardId)); + assertThat(deserialized.allocationId, equalTo(allocationId)); + if (version.onOrAfter(Version.V_7_0_0)) { // TODO update version to 6.7.0 after backport + assertThat(deserialized.primaryTerm, equalTo(primaryTerm)); + } else { + assertThat(deserialized.primaryTerm, equalTo(0L)); + } + assertThat(deserialized.message, equalTo(message)); + } + } + BytesReference serialize(Writeable writeable, Version version) throws IOException { try (BytesStreamOutput out = new BytesStreamOutput()) { out.setVersion(version); diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java b/server/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java index f248d46b11744..9b6cae43081ad 100644 --- a/server/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java +++ b/server/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java @@ -361,11 +361,14 @@ public void updateShardState(ShardRouting shardRouting, assertThat(this.shardId(), equalTo(shardRouting.shardId())); assertTrue("current: " + this.shardRouting + ", got: " + shardRouting, this.shardRouting.isSameAllocation(shardRouting)); if (this.shardRouting.active()) { - assertTrue("and active shard must stay active, current: " + this.shardRouting + ", got: " + shardRouting, + assertTrue("an active shard must stay active, current: " + this.shardRouting + ", got: " + shardRouting, shardRouting.active()); } if (this.shardRouting.primary()) { assertTrue("a primary shard can't be demoted", shardRouting.primary()); + if (this.shardRouting.initializing()) { + assertEquals("primary term can not be updated on an initializing primary shard: " + shardRouting, term, newPrimaryTerm); + } } else if (shardRouting.primary()) { // note: it's ok for a replica in post recovery to be started and promoted at once // this can happen when the primary failed after we sent the start shard message @@ -390,6 +393,10 @@ public IndexShardState state() { return null; } + public long term() { + return term; + } + public void updateTerm(long newTerm) { assertThat("term can only be incremented: " + shardRouting, newTerm, greaterThanOrEqualTo(term)); if (shardRouting.primary() && shardRouting.active()) { diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java index 8a00be28f5eb2..c1e32be9d29af 100644 --- a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java +++ b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java @@ -277,10 +277,18 @@ public ClusterState applyFailedShards(ClusterState clusterState, List startedShards) { - List entries = startedShards.stream().map(startedShard -> - new StartedShardEntry(startedShard.shardId(), startedShard.allocationId().getId(), "shard started")) - .collect(Collectors.toList()); - return runTasks(shardStartedClusterStateTaskExecutor, clusterState, entries); + final Map entries = startedShards.stream() + .collect(Collectors.toMap(Function.identity(), startedShard -> { + final IndexMetaData indexMetaData = clusterState.metaData().index(startedShard.shardId().getIndex()); + return indexMetaData != null ? indexMetaData.primaryTerm(startedShard.shardId().id()) : 0L; + })); + return applyStartedShards(clusterState, entries); + } + + public ClusterState applyStartedShards(ClusterState clusterState, Map startedShards) { + return runTasks(shardStartedClusterStateTaskExecutor, clusterState, startedShards.entrySet().stream() + .map(e -> new StartedShardEntry(e.getKey().shardId(), e.getKey().allocationId().getId(), e.getValue(), "shard started")) + .collect(Collectors.toList())); } private ClusterState runTasks(ClusterStateTaskExecutor executor, ClusterState clusterState, List entries) { diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java b/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java index b400b56b34d55..e664cc87452fc 100644 --- a/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java +++ b/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java @@ -384,7 +384,7 @@ public ClusterState randomlyUpdateClusterState(ClusterState state, } // randomly start and fail allocated shards - List startedShards = new ArrayList<>(); + final Map startedShards = new HashMap<>(); List failedShards = new ArrayList<>(); for (DiscoveryNode node : state.nodes()) { IndicesClusterStateService indicesClusterStateService = clusterStateServiceMap.get(node); @@ -393,7 +393,7 @@ public ClusterState randomlyUpdateClusterState(ClusterState state, for (MockIndexShard indexShard : indexService) { ShardRouting persistedShardRouting = indexShard.routingEntry(); if (persistedShardRouting.initializing() && randomBoolean()) { - startedShards.add(persistedShardRouting); + startedShards.put(persistedShardRouting, indexShard.term()); } else if (rarely()) { failedShards.add(new FailedShard(persistedShardRouting, "fake shard failure", new Exception(), randomBoolean())); } From 65a9b61a9162d4a9f5181b164ce4d23ee343b56e Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Tue, 29 Jan 2019 09:18:05 -0500 Subject: [PATCH 013/100] Add Seq# based optimistic concurrency control to UpdateRequest (#37872) The update request has a lesser known support for a one off update of a known document version. This PR adds an a seq# based alternative to power these operations. Relates #36148 Relates #10708 --- .../java/org/elasticsearch/client/CrudIT.java | 56 +++++++++---- docs/reference/docs/delete.asciidoc | 2 +- docs/reference/docs/index_.asciidoc | 2 +- docs/reference/docs/update.asciidoc | 8 ++ .../resources/rest-api-spec/api/update.json | 8 ++ .../test/update/35_if_seq_no.yml | 64 +++++++++++++++ .../elasticsearch/action/DocWriteRequest.java | 64 +++++++++++++++ .../action/bulk/BulkRequest.java | 1 + .../action/delete/DeleteRequest.java | 21 +---- .../action/index/IndexRequest.java | 21 +---- .../action/update/UpdateHelper.java | 4 +- .../action/update/UpdateRequest.java | 82 +++++++++++++++++++ .../action/update/UpdateRequestBuilder.java | 24 ++++++ .../elasticsearch/index/engine/Engine.java | 30 +++++++ .../index/engine/InternalEngine.java | 6 ++ .../index/get/ShardGetService.java | 24 +++--- .../action/document/RestUpdateAction.java | 2 + .../index/engine/InternalEngineTests.java | 32 ++++++++ .../index/shard/ShardGetServiceTests.java | 34 ++++++-- .../org/elasticsearch/update/UpdateIT.java | 42 ++++++++++ 20 files changed, 450 insertions(+), 77 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/update/35_if_seq_no.yml diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java index eddb6b5758452..1bf1f2487cd29 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java @@ -68,9 +68,9 @@ import org.elasticsearch.rest.action.document.RestBulkAction; import org.elasticsearch.rest.action.document.RestDeleteAction; import org.elasticsearch.rest.action.document.RestGetAction; +import org.elasticsearch.rest.action.document.RestIndexAction; import org.elasticsearch.rest.action.document.RestMultiGetAction; import org.elasticsearch.rest.action.document.RestUpdateAction; -import org.elasticsearch.rest.action.document.RestIndexAction; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; @@ -90,8 +90,10 @@ import java.util.concurrent.atomic.AtomicReference; import static java.util.Collections.singletonMap; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.lessThan; @@ -606,22 +608,46 @@ public void testUpdate() throws IOException { IndexResponse indexResponse = highLevelClient().index(indexRequest, RequestOptions.DEFAULT); assertEquals(RestStatus.CREATED, indexResponse.status()); - UpdateRequest updateRequest = new UpdateRequest("index", "id"); - updateRequest.doc(singletonMap("field", "updated"), randomFrom(XContentType.values())); - UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync); - assertEquals(RestStatus.OK, updateResponse.status()); - assertEquals(indexResponse.getVersion() + 1, updateResponse.getVersion()); - - UpdateRequest updateRequestConflict = new UpdateRequest("index", "id"); - updateRequestConflict.doc(singletonMap("field", "with_version_conflict"), randomFrom(XContentType.values())); - updateRequestConflict.version(indexResponse.getVersion()); + long lastUpdateSeqNo; + long lastUpdatePrimaryTerm; + { + UpdateRequest updateRequest = new UpdateRequest("index", "id"); + updateRequest.doc(singletonMap("field", "updated"), randomFrom(XContentType.values())); + final UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync); + assertEquals(RestStatus.OK, updateResponse.status()); + assertEquals(indexResponse.getVersion() + 1, updateResponse.getVersion()); + lastUpdateSeqNo = updateResponse.getSeqNo(); + lastUpdatePrimaryTerm = updateResponse.getPrimaryTerm(); + assertThat(lastUpdateSeqNo, greaterThanOrEqualTo(0L)); + assertThat(lastUpdatePrimaryTerm, greaterThanOrEqualTo(1L)); + } - ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () -> - execute(updateRequestConflict, highLevelClient()::update, highLevelClient()::updateAsync)); - assertEquals(RestStatus.CONFLICT, exception.status()); - assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[_doc][id]: version conflict, " + - "current version [2] is different than the one provided [1]]", exception.getMessage()); + { + final UpdateRequest updateRequest = new UpdateRequest("index", "id"); + updateRequest.doc(singletonMap("field", "with_seq_no_conflict"), randomFrom(XContentType.values())); + if (randomBoolean()) { + updateRequest.setIfSeqNo(lastUpdateSeqNo + 1); + updateRequest.setIfPrimaryTerm(lastUpdatePrimaryTerm); + } else { + updateRequest.setIfSeqNo(lastUpdateSeqNo + (randomBoolean() ? 0 : 1)); + updateRequest.setIfPrimaryTerm(lastUpdatePrimaryTerm + 1); + } + ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () -> + execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync)); + assertEquals(exception.toString(),RestStatus.CONFLICT, exception.status()); + assertThat(exception.getMessage(), containsString("Elasticsearch exception [type=version_conflict_engine_exception")); + } + { + final UpdateRequest updateRequest = new UpdateRequest("index", "id"); + updateRequest.doc(singletonMap("field", "with_seq_no"), randomFrom(XContentType.values())); + updateRequest.setIfSeqNo(lastUpdateSeqNo); + updateRequest.setIfPrimaryTerm(lastUpdatePrimaryTerm); + final UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync); + assertEquals(RestStatus.OK, updateResponse.status()); + assertEquals(lastUpdateSeqNo + 1, updateResponse.getSeqNo()); + assertEquals(lastUpdatePrimaryTerm, updateResponse.getPrimaryTerm()); + } } { IndexRequest indexRequest = new IndexRequest("index").id("with_script"); diff --git a/docs/reference/docs/delete.asciidoc b/docs/reference/docs/delete.asciidoc index bc6f7b840048d..22301b98f1031 100644 --- a/docs/reference/docs/delete.asciidoc +++ b/docs/reference/docs/delete.asciidoc @@ -39,7 +39,7 @@ The result of the above delete operation is: [[optimistic-concurrency-control-delete]] === Optimistic concurrency control -Delete operations can be made optional and only be performed if the last +Delete operations can be made conditional and only be performed if the last modification to the document was assigned the sequence number and primary term specified by the `if_seq_no` and `if_primary_term` parameters. If a mismatch is detected, the operation will result in a `VersionConflictException` diff --git a/docs/reference/docs/index_.asciidoc b/docs/reference/docs/index_.asciidoc index c7ca42bfaf4c4..257b88289d87a 100644 --- a/docs/reference/docs/index_.asciidoc +++ b/docs/reference/docs/index_.asciidoc @@ -185,7 +185,7 @@ The result of the above index operation is: [[optimistic-concurrency-control-index]] === Optimistic concurrency control -Index operations can be made optional and only be performed if the last +Index operations can be made conditional and only be performed if the last modification to the document was assigned the sequence number and primary term specified by the `if_seq_no` and `if_primary_term` parameters. If a mismatch is detected, the operation will result in a `VersionConflictException` diff --git a/docs/reference/docs/update.asciidoc b/docs/reference/docs/update.asciidoc index 1cfc122bee402..42840b1b0a5ec 100644 --- a/docs/reference/docs/update.asciidoc +++ b/docs/reference/docs/update.asciidoc @@ -349,3 +349,11 @@ version numbers being out of sync with the external system. Use the <> instead. ===================================================== + +`if_seq_no` and `if_primary_term`:: + +Update operations can be made conditional and only be performed if the last +modification to the document was assigned the sequence number and primary +term specified by the `if_seq_no` and `if_primary_term` parameters. If a +mismatch is detected, the operation will result in a `VersionConflictException` +and a status code of 409. See <> for more details. \ No newline at end of file diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/update.json b/rest-api-spec/src/main/resources/rest-api-spec/api/update.json index 37710cf07a10b..92f1013a317c3 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/update.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/update.json @@ -63,6 +63,14 @@ "type": "time", "description": "Explicit operation timeout" }, + "if_seq_no" : { + "type" : "number", + "description" : "only perform the update operation if the last operation that has changed the document has the specified sequence number" + }, + "if_primary_term" : { + "type" : "number", + "description" : "only perform the update operation if the last operation that has changed the document has the specified primary term" + }, "version": { "type": "number", "description": "Explicit version number for concurrency control" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/35_if_seq_no.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/35_if_seq_no.yml new file mode 100644 index 0000000000000..dbc569104cf4c --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/update/35_if_seq_no.yml @@ -0,0 +1,64 @@ +--- +"Update with if_seq_no": + + - skip: + version: " - 6.99.99" + reason: if_seq_no was added in 7.0 + + - do: + catch: missing + update: + index: test_1 + id: 1 + if_seq_no: 1 + if_primary_term: 1 + body: + doc: { foo: baz } + + - do: + index: + index: test_1 + id: 1 + body: + foo: baz + + - do: + catch: conflict + update: + index: test_1 + id: 1 + if_seq_no: 234 + if_primary_term: 1 + body: + doc: { foo: baz } + + - do: + update: + index: test_1 + id: 1 + if_seq_no: 0 + if_primary_term: 1 + body: + doc: { foo: bar } + + - do: + get: + index: test_1 + id: 1 + + - match: { _source: { foo: bar } } + + - do: + bulk: + body: + - update: + _index: test_1 + _id: 1 + if_seq_no: 100 + if_primary_term: 200 + - doc: + foo: baz + + - match: { errors: true } + - match: { items.0.update.status: 409 } + diff --git a/server/src/main/java/org/elasticsearch/action/DocWriteRequest.java b/server/src/main/java/org/elasticsearch/action/DocWriteRequest.java index fdf62e951a517..d8a9a3503a617 100644 --- a/server/src/main/java/org/elasticsearch/action/DocWriteRequest.java +++ b/server/src/main/java/org/elasticsearch/action/DocWriteRequest.java @@ -24,11 +24,16 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.index.VersionType; import java.io.IOException; import java.util.Locale; +import static org.elasticsearch.action.ValidateActions.addValidationError; +import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM; +import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; + /** * Generic interface to group ActionRequest, which perform writes to a single document * Action requests implementing this can be part of {@link org.elasticsearch.action.bulk.BulkRequest} @@ -117,6 +122,39 @@ public interface DocWriteRequest extends IndicesRequest { */ T versionType(VersionType versionType); + /** + * only perform this request if the document was last modification was assigned the given + * sequence number. Must be used in combination with {@link #setIfPrimaryTerm(long)} + * + * If the document last modification was assigned a different sequence number a + * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown. + */ + T setIfSeqNo(long seqNo); + + /** + * only performs this request if the document was last modification was assigned the given + * primary term. Must be used in combination with {@link #setIfSeqNo(long)} + * + * If the document last modification was assigned a different term a + * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown. + */ + T setIfPrimaryTerm(long term); + + /** + * If set, only perform this request if the document was last modification was assigned this sequence number. + * If the document last modification was assigned a different sequence number a + * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown. + */ + long ifSeqNo(); + + /** + * If set, only perform this request if the document was last modification was assigned this primary term. + * + * If the document last modification was assigned a different term a + * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown. + */ + long ifPrimaryTerm(); + /** * Get the requested document operation type of the request * @return the operation type {@link OpType} @@ -216,4 +254,30 @@ static void writeDocumentRequest(StreamOutput out, DocWriteRequest request) throw new IllegalStateException("invalid request [" + request.getClass().getSimpleName() + " ]"); } } + + static ActionRequestValidationException validateSeqNoBasedCASParams( + DocWriteRequest request, ActionRequestValidationException validationException) { + if (request.versionType().validateVersionForWrites(request.version()) == false) { + validationException = addValidationError("illegal version value [" + request.version() + "] for version type [" + + request.versionType().name() + "]", validationException); + } + if (request.versionType() == VersionType.FORCE) { + validationException = addValidationError("version type [force] may no longer be used", validationException); + } + + if (request.ifSeqNo() != UNASSIGNED_SEQ_NO && ( + request.versionType() != VersionType.INTERNAL || request.version() != Versions.MATCH_ANY + )) { + validationException = addValidationError("compare and write operations can not use versioning", validationException); + } + if (request.ifPrimaryTerm() == UNASSIGNED_PRIMARY_TERM && request.ifSeqNo() != UNASSIGNED_SEQ_NO) { + validationException = addValidationError("ifSeqNo is set, but primary term is [0]", validationException); + } + if (request.ifPrimaryTerm() != UNASSIGNED_PRIMARY_TERM && request.ifSeqNo() == UNASSIGNED_SEQ_NO) { + validationException = + addValidationError("ifSeqNo is unassigned, but primary term is [" + request.ifPrimaryTerm() + "]", validationException); + } + + return validationException; + } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java index e58b0dfbffbd3..b5c786ab2df6d 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java @@ -503,6 +503,7 @@ public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Null } else if ("update".equals(action)) { UpdateRequest updateRequest = new UpdateRequest(index, type, id).routing(routing).retryOnConflict(retryOnConflict) .version(version).versionType(versionType) + .setIfSeqNo(ifSeqNo).setIfPrimaryTerm(ifPrimaryTerm) .routing(routing); // EMPTY is safe here because we never call namedObject try (InputStream dataStream = sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType).streamInput(); diff --git a/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java b/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java index 8d2967fd28ba4..a033bf3cb000f 100644 --- a/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java +++ b/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java @@ -110,27 +110,8 @@ public ActionRequestValidationException validate() { if (Strings.isEmpty(id)) { validationException = addValidationError("id is missing", validationException); } - if (versionType.validateVersionForWrites(version) == false) { - validationException = addValidationError("illegal version value [" + version + "] for version type [" - + versionType.name() + "]", validationException); - } - if (versionType == VersionType.FORCE) { - validationException = addValidationError("version type [force] may no longer be used", validationException); - } - - if (ifSeqNo != UNASSIGNED_SEQ_NO && ( - versionType != VersionType.INTERNAL || version != Versions.MATCH_ANY - )) { - validationException = addValidationError("compare and write operations can not use versioning", validationException); - } - if (ifPrimaryTerm == UNASSIGNED_PRIMARY_TERM && ifSeqNo != UNASSIGNED_SEQ_NO) { - validationException = addValidationError("ifSeqNo is set, but primary term is [0]", validationException); - } - if (ifPrimaryTerm != UNASSIGNED_PRIMARY_TERM && ifSeqNo == UNASSIGNED_SEQ_NO) { - validationException = - addValidationError("ifSeqNo is unassigned, but primary term is [" + ifPrimaryTerm + "]", validationException); - } + validationException = DocWriteRequest.validateSeqNoBasedCASParams(this, validationException); return validationException; } diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java index a9aac3025de1e..37d960831776d 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -188,14 +188,7 @@ public ActionRequestValidationException validate() { addValidationError("an id is required for a " + opType() + " operation", validationException); } - if (!versionType.validateVersionForWrites(resolvedVersion)) { - validationException = addValidationError("illegal version value [" + resolvedVersion + "] for version type [" - + versionType.name() + "]", validationException); - } - - if (versionType == VersionType.FORCE) { - validationException = addValidationError("version type [force] may no longer be used", validationException); - } + validationException = DocWriteRequest.validateSeqNoBasedCASParams(this, validationException); if (id != null && id.getBytes(StandardCharsets.UTF_8).length > 512) { validationException = addValidationError("id is too long, must be no longer than 512 bytes but was: " + @@ -210,18 +203,6 @@ public ActionRequestValidationException validate() { validationException = addValidationError("pipeline cannot be an empty string", validationException); } - if (ifSeqNo != UNASSIGNED_SEQ_NO && ( - versionType != VersionType.INTERNAL || version != Versions.MATCH_ANY - )) { - validationException = addValidationError("compare and write operations can not use versioning", validationException); - } - if (ifPrimaryTerm == UNASSIGNED_PRIMARY_TERM && ifSeqNo != UNASSIGNED_SEQ_NO) { - validationException = addValidationError("ifSeqNo is set, but primary term is [0]", validationException); - } - if (ifPrimaryTerm != UNASSIGNED_PRIMARY_TERM && ifSeqNo == UNASSIGNED_SEQ_NO) { - validationException = - addValidationError("ifSeqNo is unassigned, but primary term is [" + ifPrimaryTerm + "]", validationException); - } return validationException; } diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java b/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java index a8a5fb8f72f30..8cd6146768fff 100644 --- a/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java +++ b/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java @@ -69,8 +69,8 @@ public UpdateHelper(ScriptService scriptService) { * Prepares an update request by converting it into an index or delete request or an update response (no action). */ public Result prepare(UpdateRequest request, IndexShard indexShard, LongSupplier nowInMillis) { - final GetResult getResult = indexShard.getService().getForUpdate(request.type(), request.id(), request.version(), - request.versionType()); + final GetResult getResult = indexShard.getService().getForUpdate( + request.type(), request.id(), request.version(), request.versionType(), request.ifSeqNo(), request.ifPrimaryTerm()); return prepare(indexShard.shardId(), request, getResult, nowInMillis); } diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java b/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java index a7805b4cbdbad..2a1865aa80818 100644 --- a/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java @@ -54,6 +54,8 @@ import java.util.Map; import static org.elasticsearch.action.ValidateActions.addValidationError; +import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM; +import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; public class UpdateRequest extends InstanceShardOperationRequest implements DocWriteRequest, WriteRequest, ToXContentObject { @@ -66,6 +68,8 @@ public class UpdateRequest extends InstanceShardOperationRequest private static final ParseField DOC_AS_UPSERT_FIELD = new ParseField("doc_as_upsert"); private static final ParseField DETECT_NOOP_FIELD = new ParseField("detect_noop"); private static final ParseField SOURCE_FIELD = new ParseField("_source"); + private static final ParseField IF_SEQ_NO = new ParseField("if_seq_no"); + private static final ParseField IF_PRIMARY_TERM = new ParseField("if_primary_term"); static { PARSER = new ObjectParser<>(UpdateRequest.class.getSimpleName()); @@ -89,6 +93,8 @@ public class UpdateRequest extends InstanceShardOperationRequest PARSER.declareField(UpdateRequest::fetchSource, (parser, context) -> FetchSourceContext.fromXContent(parser), SOURCE_FIELD, ObjectParser.ValueType.OBJECT_ARRAY_BOOLEAN_OR_STRING); + PARSER.declareLong(UpdateRequest::setIfSeqNo, IF_SEQ_NO); + PARSER.declareLong(UpdateRequest::setIfPrimaryTerm, IF_PRIMARY_TERM); } // Set to null initially so we can know to override in bulk requests that have a default type. @@ -105,6 +111,9 @@ public class UpdateRequest extends InstanceShardOperationRequest private long version = Versions.MATCH_ANY; private VersionType versionType = VersionType.INTERNAL; private int retryOnConflict = 0; + private long ifSeqNo = UNASSIGNED_SEQ_NO; + private long ifPrimaryTerm = UNASSIGNED_PRIMARY_TERM; + private RefreshPolicy refreshPolicy = RefreshPolicy.NONE; @@ -170,6 +179,16 @@ public ActionRequestValidationException validate() { } } + validationException = DocWriteRequest.validateSeqNoBasedCASParams(this, validationException); + + if (ifSeqNo != UNASSIGNED_SEQ_NO && retryOnConflict > 0) { + validationException = addValidationError("compare and write operations can not be retried", validationException); + } + + if (ifSeqNo != UNASSIGNED_SEQ_NO && docAsUpsert) { + validationException = addValidationError("compare and write operations can not be used with upsert", validationException); + } + if (script == null && doc == null) { validationException = addValidationError("script or doc is missing", validationException); } @@ -531,6 +550,55 @@ public VersionType versionType() { return this.versionType; } + /** + * only perform this update request if the document's modification was assigned the given + * sequence number. Must be used in combination with {@link #setIfPrimaryTerm(long)} + * + * If the document last modification was assigned a different sequence number a + * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown. + */ + public UpdateRequest setIfSeqNo(long seqNo) { + if (seqNo < 0 && seqNo != UNASSIGNED_SEQ_NO) { + throw new IllegalArgumentException("sequence numbers must be non negative. got [" + seqNo + "]."); + } + ifSeqNo = seqNo; + return this; + } + + /** + * only performs this update request if the document's last modification was assigned the given + * primary term. Must be used in combination with {@link #setIfSeqNo(long)} + * + * If the document last modification was assigned a different term a + * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown. + */ + public UpdateRequest setIfPrimaryTerm(long term) { + if (term < 0) { + throw new IllegalArgumentException("primary term must be non negative. got [" + term + "]"); + } + ifPrimaryTerm = term; + return this; + } + + /** + * If set, only perform this update request if the document was last modification was assigned this sequence number. + * If the document last modification was assigned a different sequence number a + * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown. + */ + public long ifSeqNo() { + return ifSeqNo; + } + + /** + * If set, only perform this update request if the document was last modification was assigned this primary term. + * + * If the document last modification was assigned a different term a + * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown. + */ + public long ifPrimaryTerm() { + return ifPrimaryTerm; + } + @Override public OpType opType() { return OpType.UPDATE; @@ -811,6 +879,10 @@ public void readFrom(StreamInput in) throws IOException { docAsUpsert = in.readBoolean(); version = in.readLong(); versionType = VersionType.fromValue(in.readByte()); + if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + ifSeqNo = in.readZLong(); + ifPrimaryTerm = in.readVLong(); + } detectNoop = in.readBoolean(); scriptedUpsert = in.readBoolean(); } @@ -862,6 +934,10 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(docAsUpsert); out.writeLong(version); out.writeByte(versionType.getValue()); + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + out.writeZLong(ifSeqNo); + out.writeVLong(ifPrimaryTerm); + } out.writeBoolean(detectNoop); out.writeBoolean(scriptedUpsert); } @@ -880,6 +956,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.copyCurrentStructure(parser); } } + + if (ifSeqNo != UNASSIGNED_SEQ_NO) { + builder.field(IF_SEQ_NO.getPreferredName(), ifSeqNo); + builder.field(IF_PRIMARY_TERM.getPreferredName(), ifPrimaryTerm); + } + if (script != null) { builder.field("script", script); } diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java index 181dba6a10734..919f460e8c07b 100644 --- a/server/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java @@ -150,6 +150,30 @@ public UpdateRequestBuilder setVersionType(VersionType versionType) { return this; } + /** + * only perform this update request if the document was last modification was assigned the given + * sequence number. Must be used in combination with {@link #setIfPrimaryTerm(long)} + * + * If the document last modification was assigned a different sequence number a + * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown. + */ + public UpdateRequestBuilder setIfSeqNo(long seqNo) { + request.setIfSeqNo(seqNo); + return this; + } + + /** + * only perform this update request if the document was last modification was assigned the given + * primary term. Must be used in combination with {@link #setIfSeqNo(long)} + * + * If the document last modification was assigned a different term a + * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown. + */ + public UpdateRequestBuilder setIfPrimaryTerm(long term) { + request.setIfPrimaryTerm(term); + return this; + } + /** * Sets the number of shard copies that must be active before proceeding with the write. * See {@link ReplicationRequest#waitForActiveShards(ActiveShardCount)} for details. diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java index e1df104d338df..e450e93e9d397 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -72,6 +72,7 @@ import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.merge.MergeStats; import org.elasticsearch.index.seqno.SeqNoStats; +import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.DocsStats; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.store.Store; @@ -622,6 +623,13 @@ protected final GetResult getFromSearcher(Get get, BiFunction search throw new VersionConflictEngineException(shardId, get.type(), get.id(), get.versionType().explainConflictForReads(versionValue.version, get.version())); } + if (get.getIfSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO && ( + get.getIfSeqNo() != versionValue.seqNo || get.getIfPrimaryTerm() != versionValue.term + )) { + throw new VersionConflictEngineException(shardId, get.type(), get.id(), + get.getIfSeqNo(), get.getIfPrimaryTerm(), versionValue.seqNo, versionValue.term); + } if (get.isReadFromTranslog()) { // this is only used for updates - API _GET calls will always read form a reader for consistency // the update call doesn't need the consistency since it's source only + _parent but parent can go away in 7.0 diff --git a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java index 6d58b981ddc53..9fb1cb804946f 100644 --- a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java +++ b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java @@ -45,7 +45,6 @@ import org.elasticsearch.index.mapper.RoutingFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.Uid; -import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.AbstractIndexShardComponent; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; @@ -56,6 +55,9 @@ import java.util.Map; import java.util.concurrent.TimeUnit; +import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM; +import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; + public final class ShardGetService extends AbstractIndexShardComponent { private final MapperService mapperService; private final MeanMetric existsMetric = new MeanMetric(); @@ -77,15 +79,17 @@ public GetStats stats() { public GetResult get(String type, String id, String[] gFields, boolean realtime, long version, VersionType versionType, FetchSourceContext fetchSourceContext) { - return get(type, id, gFields, realtime, version, versionType, fetchSourceContext, false); + return + get(type, id, gFields, realtime, version, versionType, UNASSIGNED_SEQ_NO, UNASSIGNED_PRIMARY_TERM, fetchSourceContext, false); } private GetResult get(String type, String id, String[] gFields, boolean realtime, long version, VersionType versionType, - FetchSourceContext fetchSourceContext, boolean readFromTranslog) { + long ifSeqNo, long ifPrimaryTerm, FetchSourceContext fetchSourceContext, boolean readFromTranslog) { currentMetric.inc(); try { long now = System.nanoTime(); - GetResult getResult = innerGet(type, id, gFields, realtime, version, versionType, fetchSourceContext, readFromTranslog); + GetResult getResult = + innerGet(type, id, gFields, realtime, version, versionType, ifSeqNo, ifPrimaryTerm, fetchSourceContext, readFromTranslog); if (getResult.isExists()) { existsMetric.inc(System.nanoTime() - now); @@ -98,8 +102,8 @@ private GetResult get(String type, String id, String[] gFields, boolean realtime } } - public GetResult getForUpdate(String type, String id, long version, VersionType versionType) { - return get(type, id, new String[]{RoutingFieldMapper.NAME}, true, version, versionType, + public GetResult getForUpdate(String type, String id, long version, VersionType versionType, long ifSeqNo, long ifPrimaryTerm) { + return get(type, id, new String[]{RoutingFieldMapper.NAME}, true, version, versionType, ifSeqNo, ifPrimaryTerm, FetchSourceContext.FETCH_SOURCE, true); } @@ -113,7 +117,7 @@ public GetResult getForUpdate(String type, String id, long version, VersionType public GetResult get(Engine.GetResult engineGetResult, String id, String type, String[] fields, FetchSourceContext fetchSourceContext) { if (!engineGetResult.exists()) { - return new GetResult(shardId.getIndexName(), type, id, SequenceNumbers.UNASSIGNED_SEQ_NO, 0, -1, false, null, null); + return new GetResult(shardId.getIndexName(), type, id, UNASSIGNED_SEQ_NO, UNASSIGNED_PRIMARY_TERM, -1, false, null, null); } currentMetric.inc(); @@ -151,7 +155,7 @@ private FetchSourceContext normalizeFetchSourceContent(@Nullable FetchSourceCont } private GetResult innerGet(String type, String id, String[] gFields, boolean realtime, long version, VersionType versionType, - FetchSourceContext fetchSourceContext, boolean readFromTranslog) { + long ifSeqNo, long ifPrimaryTerm, FetchSourceContext fetchSourceContext, boolean readFromTranslog) { fetchSourceContext = normalizeFetchSourceContent(fetchSourceContext, gFields); if (type == null || type.equals("_all")) { DocumentMapper mapper = mapperService.documentMapper(); @@ -162,14 +166,14 @@ private GetResult innerGet(String type, String id, String[] gFields, boolean rea if (type != null) { Term uidTerm = new Term(IdFieldMapper.NAME, Uid.encodeId(id)); get = indexShard.get(new Engine.Get(realtime, readFromTranslog, type, id, uidTerm) - .version(version).versionType(versionType)); + .version(version).versionType(versionType).setIfSeqNo(ifSeqNo).setIfPrimaryTerm(ifPrimaryTerm)); if (get.exists() == false) { get.close(); } } if (get == null || get.exists() == false) { - return new GetResult(shardId.getIndexName(), type, id, SequenceNumbers.UNASSIGNED_SEQ_NO, 0, -1, false, null, null); + return new GetResult(shardId.getIndexName(), type, id, UNASSIGNED_SEQ_NO, UNASSIGNED_PRIMARY_TERM, -1, false, null, null); } try { diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java index 033176c4a7300..463a18ea6b802 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java @@ -86,6 +86,8 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC updateRequest.version(RestActions.parseVersion(request)); updateRequest.versionType(VersionType.fromString(request.param("version_type"), updateRequest.versionType())); + updateRequest.setIfSeqNo(request.paramAsLong("if_seq_no", updateRequest.ifSeqNo())); + updateRequest.setIfPrimaryTerm(request.paramAsLong("if_primary_term", updateRequest.ifPrimaryTerm())); request.applyContentParser(parser -> { updateRequest.fromXContent(parser); diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index edf1925fdd798..f57bff72fc57c 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -1378,6 +1378,38 @@ public void testVersionedUpdate() throws IOException { } + public void testGetIfSeqNoIfPrimaryTerm() throws IOException { + final BiFunction searcherFactory = engine::acquireSearcher; + + ParsedDocument doc = testParsedDocument("1", null, testDocument(), B_1, null); + Engine.Index create = new Engine.Index(newUid(doc), primaryTerm.get(), doc, Versions.MATCH_DELETED); + Engine.IndexResult indexResult = engine.index(create); + if (randomBoolean()) { + engine.refresh("test"); + } + if (randomBoolean()) { + engine.flush(); + } + try (Engine.GetResult get = engine.get( + new Engine.Get(true, true, doc.type(), doc.id(), create.uid()) + .setIfSeqNo(indexResult.getSeqNo()).setIfPrimaryTerm(primaryTerm.get()), + searcherFactory)) { + assertEquals(indexResult.getSeqNo(), get.docIdAndVersion().seqNo); + } + + expectThrows(VersionConflictEngineException.class, () -> engine.get(new Engine.Get(true, false, doc.type(), doc.id(), create.uid()) + .setIfSeqNo(indexResult.getSeqNo() + 1).setIfPrimaryTerm(primaryTerm.get()), + searcherFactory)); + + expectThrows(VersionConflictEngineException.class, () -> engine.get(new Engine.Get(true, false, doc.type(), doc.id(), create.uid()) + .setIfSeqNo(indexResult.getSeqNo()).setIfPrimaryTerm(primaryTerm.get() + 1), + searcherFactory)); + + expectThrows(VersionConflictEngineException.class, () -> engine.get(new Engine.Get(true, false, doc.type(), doc.id(), create.uid()) + .setIfSeqNo(indexResult.getSeqNo() + 1).setIfPrimaryTerm(primaryTerm.get() + 1), + searcherFactory)); + } + public void testVersioningNewIndex() throws IOException { ParsedDocument doc = testParsedDocument("1", null, testDocument(), B_1, null); Engine.Index index = indexForDoc(doc); diff --git a/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java b/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java index 7db904f89dfa8..14e513ff89cfe 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java @@ -20,17 +20,21 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.get.GetResult; import org.elasticsearch.index.mapper.RoutingFieldMapper; import java.io.IOException; import java.nio.charset.StandardCharsets; +import static org.elasticsearch.common.lucene.uid.Versions.MATCH_ANY; +import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM; +import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; + public class ShardGetServiceTests extends IndexShardTestCase { public void testGetForUpdate() throws IOException { @@ -47,7 +51,8 @@ public void testGetForUpdate() throws IOException { recoverShardFromStore(primary); Engine.IndexResult test = indexDoc(primary, "test", "0", "{\"foo\" : \"bar\"}"); assertTrue(primary.getEngine().refreshNeeded()); - GetResult testGet = primary.getService().getForUpdate("test", "0", test.getVersion(), VersionType.INTERNAL); + GetResult testGet = primary.getService().getForUpdate( + "test", "0", test.getVersion(), VersionType.INTERNAL, UNASSIGNED_SEQ_NO, UNASSIGNED_PRIMARY_TERM); assertFalse(testGet.getFields().containsKey(RoutingFieldMapper.NAME)); assertEquals(new String(testGet.source(), StandardCharsets.UTF_8), "{\"foo\" : \"bar\"}"); try (Engine.Searcher searcher = primary.getEngine().acquireSearcher("test", Engine.SearcherScope.INTERNAL)) { @@ -56,7 +61,8 @@ public void testGetForUpdate() throws IOException { Engine.IndexResult test1 = indexDoc(primary, "test", "1", "{\"foo\" : \"baz\"}", XContentType.JSON, "foobar"); assertTrue(primary.getEngine().refreshNeeded()); - GetResult testGet1 = primary.getService().getForUpdate("test", "1", test1.getVersion(), VersionType.INTERNAL); + GetResult testGet1 = primary.getService().getForUpdate( + "test", "1", test1.getVersion(), VersionType.INTERNAL, UNASSIGNED_SEQ_NO, UNASSIGNED_PRIMARY_TERM); assertEquals(new String(testGet1.source(), StandardCharsets.UTF_8), "{\"foo\" : \"baz\"}"); assertTrue(testGet1.getFields().containsKey(RoutingFieldMapper.NAME)); assertEquals("foobar", testGet1.getFields().get(RoutingFieldMapper.NAME).getValue()); @@ -69,13 +75,22 @@ public void testGetForUpdate() throws IOException { } // now again from the reader - test1 = indexDoc(primary, "test", "1", "{\"foo\" : \"baz\"}", XContentType.JSON, "foobar"); + Engine.IndexResult test2 = indexDoc(primary, "test", "1", "{\"foo\" : \"baz\"}", XContentType.JSON, "foobar"); assertTrue(primary.getEngine().refreshNeeded()); - testGet1 = primary.getService().getForUpdate("test", "1", test1.getVersion(), VersionType.INTERNAL); + testGet1 = primary.getService().getForUpdate("test", "1", test2.getVersion(), VersionType.INTERNAL, + UNASSIGNED_SEQ_NO, UNASSIGNED_PRIMARY_TERM); assertEquals(new String(testGet1.source(), StandardCharsets.UTF_8), "{\"foo\" : \"baz\"}"); assertTrue(testGet1.getFields().containsKey(RoutingFieldMapper.NAME)); assertEquals("foobar", testGet1.getFields().get(RoutingFieldMapper.NAME).getValue()); + final long primaryTerm = primary.operationPrimaryTerm; + testGet1 = primary.getService().getForUpdate("test", "1", MATCH_ANY, VersionType.INTERNAL, test2.getSeqNo(), primaryTerm); + assertEquals(new String(testGet1.source(), StandardCharsets.UTF_8), "{\"foo\" : \"baz\"}"); + + expectThrows(VersionConflictEngineException.class, () -> + primary.getService().getForUpdate("test", "1", MATCH_ANY, VersionType.INTERNAL, test2.getSeqNo() + 1, primaryTerm)); + expectThrows(VersionConflictEngineException.class, () -> + primary.getService().getForUpdate("test", "1", MATCH_ANY, VersionType.INTERNAL, test2.getSeqNo(), primaryTerm + 1)); closeShards(primary); } @@ -93,13 +108,16 @@ public void testTypelessGetForUpdate() throws IOException { Engine.IndexResult indexResult = indexDoc(shard, "some_type", "0", "{\"foo\" : \"bar\"}"); assertTrue(indexResult.isCreated()); - GetResult getResult = shard.getService().getForUpdate("some_type", "0", Versions.MATCH_ANY, VersionType.INTERNAL); + GetResult getResult = shard.getService().getForUpdate( + "some_type", "0", MATCH_ANY, VersionType.INTERNAL, UNASSIGNED_SEQ_NO, UNASSIGNED_PRIMARY_TERM); assertTrue(getResult.isExists()); - getResult = shard.getService().getForUpdate("some_other_type", "0", Versions.MATCH_ANY, VersionType.INTERNAL); + getResult = shard.getService().getForUpdate( + "some_other_type", "0", MATCH_ANY, VersionType.INTERNAL, UNASSIGNED_SEQ_NO, UNASSIGNED_PRIMARY_TERM); assertFalse(getResult.isExists()); - getResult = shard.getService().getForUpdate("_doc", "0", Versions.MATCH_ANY, VersionType.INTERNAL); + getResult = shard.getService().getForUpdate( + "_doc", "0", MATCH_ANY, VersionType.INTERNAL, UNASSIGNED_SEQ_NO, UNASSIGNED_PRIMARY_TERM); assertTrue(getResult.isExists()); closeShards(shard); diff --git a/server/src/test/java/org/elasticsearch/update/UpdateIT.java b/server/src/test/java/org/elasticsearch/update/UpdateIT.java index 05b27758ee434..7652c503450ae 100644 --- a/server/src/test/java/org/elasticsearch/update/UpdateIT.java +++ b/server/src/test/java/org/elasticsearch/update/UpdateIT.java @@ -27,6 +27,7 @@ import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.action.update.UpdateResponse; @@ -36,7 +37,9 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.MergePolicyConfig; import org.elasticsearch.index.engine.DocumentMissingException; +import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.script.MockScriptPlugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; @@ -411,6 +414,45 @@ public void testUpdate() throws Exception { } } + public void testUpdateWithIfSeqNo() throws Exception { + createTestIndex(); + ensureGreen(); + + IndexResponse result = client().prepareIndex("test", "type1", "1").setSource("field", 1).get(); + expectThrows(VersionConflictEngineException.class, () -> + client().prepareUpdate(indexOrAlias(), "type1", "1") + .setDoc(XContentFactory.jsonBuilder().startObject().field("field", 2).endObject()) + .setIfSeqNo(result.getSeqNo() + 1) + .setIfPrimaryTerm(result.getPrimaryTerm()) + .get() + ); + + expectThrows(VersionConflictEngineException.class, () -> + client().prepareUpdate(indexOrAlias(), "type1", "1") + .setDoc(XContentFactory.jsonBuilder().startObject().field("field", 2).endObject()) + .setIfSeqNo(result.getSeqNo()) + .setIfPrimaryTerm(result.getPrimaryTerm() + 1) + .get() + ); + + expectThrows(VersionConflictEngineException.class, () -> + client().prepareUpdate(indexOrAlias(), "type1", "1") + .setDoc(XContentFactory.jsonBuilder().startObject().field("field", 2).endObject()) + .setIfSeqNo(result.getSeqNo() + 1) + .setIfPrimaryTerm(result.getPrimaryTerm() + 1) + .get() + ); + + UpdateResponse updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1") + .setDoc(XContentFactory.jsonBuilder().startObject().field("field", 2).endObject()) + .setIfSeqNo(result.getSeqNo()) + .setIfPrimaryTerm(result.getPrimaryTerm()) + .get(); + + assertThat(updateResponse.status(), equalTo(RestStatus.OK)); + assertThat(updateResponse.getSeqNo(), equalTo(result.getSeqNo() + 1)); + } + public void testUpdateRequestWithBothScriptAndDoc() throws Exception { createTestIndex(); ensureGreen(); From 09a11a34effe7e9f81157560f5936b2790c50201 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Tue, 29 Jan 2019 15:31:49 +0100 Subject: [PATCH 014/100] Remove clusterAlias instance member from QueryShardContext (#37923) The clusterAlias member is only used in the copy constructor, to be able to reconstruct the fully qualified index. It is also possible to remove the instance member and add a private constructor that accepts the already built Index object which contains the cluster alias. --- .../index/query/QueryShardContext.java | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index c398fde04a2f6..2b5415115895d 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -86,7 +86,6 @@ public class QueryShardContext extends QueryRewriteContext { private final BiFunction> indexFieldDataService; private final int shardId; private final IndexReader reader; - private final String clusterAlias; private String[] types = Strings.EMPTY_ARRAY; private boolean cacheable = true; private final SetOnce frozen = new SetOnce<>(); @@ -110,6 +109,23 @@ public QueryShardContext(int shardId, IndexSettings indexSettings, BitsetFilterC SimilarityService similarityService, ScriptService scriptService, NamedXContentRegistry xContentRegistry, NamedWriteableRegistry namedWriteableRegistry, Client client, IndexReader reader, LongSupplier nowInMillis, String clusterAlias) { + this(shardId, indexSettings, bitsetFilterCache, indexFieldDataLookup, mapperService, similarityService, scriptService, + xContentRegistry, namedWriteableRegistry, client, reader, nowInMillis, new Index(RemoteClusterAware.buildRemoteIndexName( + clusterAlias, indexSettings.getIndex().getName()), indexSettings.getIndex().getUUID())); + } + + public QueryShardContext(QueryShardContext source) { + this(source.shardId, source.indexSettings, source.bitsetFilterCache, source.indexFieldDataService, source.mapperService, + source.similarityService, source.scriptService, source.getXContentRegistry(), source.getWriteableRegistry(), + source.client, source.reader, source.nowInMillis, source.fullyQualifiedIndex); + this.types = source.getTypes(); + } + + private QueryShardContext(int shardId, IndexSettings indexSettings, BitsetFilterCache bitsetFilterCache, + BiFunction> indexFieldDataLookup, MapperService mapperService, + SimilarityService similarityService, ScriptService scriptService, NamedXContentRegistry xContentRegistry, + NamedWriteableRegistry namedWriteableRegistry, Client client, IndexReader reader, LongSupplier nowInMillis, + Index fullyQualifiedIndex) { super(xContentRegistry, namedWriteableRegistry,client, nowInMillis); this.shardId = shardId; this.similarityService = similarityService; @@ -121,16 +137,7 @@ public QueryShardContext(int shardId, IndexSettings indexSettings, BitsetFilterC this.scriptService = scriptService; this.indexSettings = indexSettings; this.reader = reader; - this.clusterAlias = clusterAlias; - this.fullyQualifiedIndex = new Index(RemoteClusterAware.buildRemoteIndexName(clusterAlias, indexSettings.getIndex().getName()), - indexSettings.getIndex().getUUID()); - } - - public QueryShardContext(QueryShardContext source) { - this(source.shardId, source.indexSettings, source.bitsetFilterCache, source.indexFieldDataService, source.mapperService, - source.similarityService, source.scriptService, source.getXContentRegistry(), source.getWriteableRegistry(), - source.client, source.reader, source.nowInMillis, source.clusterAlias); - this.types = source.getTypes(); + this.fullyQualifiedIndex = fullyQualifiedIndex; } private void reset() { From 3c9f7031b92dab442305eb9bbc1df48e5d4aa6da Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Tue, 29 Jan 2019 15:41:05 +0100 Subject: [PATCH 015/100] Enforce cluster UUIDs (#37775) This commit adds join validation around cluster UUIDs, preventing a node to join a cluster if it was previously part of another cluster. The commit introduces a new flag to the cluster state, clusterUUIDCommitted, which denotes whether the node has locked into a cluster with the given uuid. When a cluster is committed, this flag will turn to true, and subsequent cluster state updates will keep the information about committal. Note that coordinating-only nodes are still free to switch clusters at will (after restart), as they don't carry any persistent state. --- .../coordination/CoordinationState.java | 26 ++++- .../cluster/coordination/Coordinator.java | 16 ++- .../cluster/coordination/JoinHelper.java | 10 +- .../cluster/metadata/MetaData.java | 55 ++++++++-- .../coordination/CoordinatorTests.java | 101 ++++++++++++++++-- .../cluster/coordination/JoinHelperTests.java | 2 +- .../cluster/metadata/MetaDataTests.java | 37 +++++++ .../discovery/ClusterDisruptionIT.java | 28 +++++ .../gateway/ClusterStateUpdatersTests.java | 6 +- .../GatewayMetaStatePersistedStateTests.java | 10 +- 10 files changed, 261 insertions(+), 30 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java index 4d542566ccd70..dff6b5add0b09 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java @@ -422,7 +422,7 @@ public void handleCommit(ApplyCommitRequest applyCommit) { logger.trace("handleCommit: applying commit request for term [{}] and version [{}]", applyCommit.getTerm(), applyCommit.getVersion()); - persistedState.markLastAcceptedConfigAsCommitted(); + persistedState.markLastAcceptedStateAsCommitted(); assert getLastCommittedConfiguration().equals(getLastAcceptedConfiguration()); } @@ -471,16 +471,32 @@ public interface PersistedState { /** * Marks the last accepted cluster state as committed. * After a successful call to this method, {@link #getLastAcceptedState()} should return the last cluster state that was set, - * with the last committed configuration now corresponding to the last accepted configuration. + * with the last committed configuration now corresponding to the last accepted configuration, and the cluster uuid, if set, + * marked as committed. */ - default void markLastAcceptedConfigAsCommitted() { + default void markLastAcceptedStateAsCommitted() { final ClusterState lastAcceptedState = getLastAcceptedState(); + MetaData.Builder metaDataBuilder = null; if (lastAcceptedState.getLastAcceptedConfiguration().equals(lastAcceptedState.getLastCommittedConfiguration()) == false) { final CoordinationMetaData coordinationMetaData = CoordinationMetaData.builder(lastAcceptedState.coordinationMetaData()) .lastCommittedConfiguration(lastAcceptedState.getLastAcceptedConfiguration()) .build(); - final MetaData metaData = MetaData.builder(lastAcceptedState.metaData()).coordinationMetaData(coordinationMetaData).build(); - setLastAcceptedState(ClusterState.builder(lastAcceptedState).metaData(metaData).build()); + metaDataBuilder = MetaData.builder(lastAcceptedState.metaData()); + metaDataBuilder.coordinationMetaData(coordinationMetaData); + } + // if we receive a commit from a Zen1 master that has not recovered its state yet, the cluster uuid might not been known yet. + assert lastAcceptedState.metaData().clusterUUID().equals(MetaData.UNKNOWN_CLUSTER_UUID) == false || + lastAcceptedState.term() == ZEN1_BWC_TERM : + "received cluster state with empty cluster uuid but not Zen1 BWC term: " + lastAcceptedState; + if (lastAcceptedState.metaData().clusterUUID().equals(MetaData.UNKNOWN_CLUSTER_UUID) == false && + lastAcceptedState.metaData().clusterUUIDCommitted() == false) { + if (metaDataBuilder == null) { + metaDataBuilder = MetaData.builder(lastAcceptedState.metaData()); + } + metaDataBuilder.clusterUUIDCommitted(true); + } + if (metaDataBuilder != null) { + setLastAcceptedState(ClusterState.builder(lastAcceptedState).metaData(metaDataBuilder).build()); } } } diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java index 4bf977f8398ce..dff9cdcb8a2a5 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java @@ -147,7 +147,7 @@ public Coordinator(String nodeName, Settings settings, ClusterSettings clusterSe this.masterService = masterService; this.onJoinValidators = JoinTaskExecutor.addBuiltInJoinValidators(onJoinValidators); this.joinHelper = new JoinHelper(settings, allocationService, masterService, transportService, - this::getCurrentTerm, this::handleJoinRequest, this::joinLeaderInTerm, this.onJoinValidators); + this::getCurrentTerm, this::getStateForMasterService, this::handleJoinRequest, this::joinLeaderInTerm, this.onJoinValidators); this.persistedStateSupplier = persistedStateSupplier; this.discoverySettings = new DiscoverySettings(settings, clusterSettings); this.lastKnownLeader = Optional.empty(); @@ -281,7 +281,18 @@ PublishWithJoinResponse handlePublishRequest(PublishRequest publishRequest) { + lastKnownLeader + ", rejecting"); } - if (publishRequest.getAcceptedState().term() > coordinationState.get().getLastAcceptedState().term()) { + final ClusterState localState = coordinationState.get().getLastAcceptedState(); + + if (localState.metaData().clusterUUIDCommitted() && + localState.metaData().clusterUUID().equals(publishRequest.getAcceptedState().metaData().clusterUUID()) == false) { + logger.warn("received cluster state from {} with a different cluster uuid {} than local cluster uuid {}, rejecting", + sourceNode, publishRequest.getAcceptedState().metaData().clusterUUID(), localState.metaData().clusterUUID()); + throw new CoordinationStateRejectedException("received cluster state from " + sourceNode + + " with a different cluster uuid " + publishRequest.getAcceptedState().metaData().clusterUUID() + + " than local cluster uuid " + localState.metaData().clusterUUID() + ", rejecting"); + } + + if (publishRequest.getAcceptedState().term() > localState.term()) { // only do join validation if we have not accepted state from this master yet onJoinValidators.forEach(a -> a.accept(getLocalNode(), publishRequest.getAcceptedState())); } @@ -653,6 +664,7 @@ public void invariant() { assert followersChecker.getFastResponseState().term == getCurrentTerm() : followersChecker.getFastResponseState(); assert followersChecker.getFastResponseState().mode == getMode() : followersChecker.getFastResponseState(); assert (applierState.nodes().getMasterNodeId() == null) == applierState.blocks().hasGlobalBlockWithId(NO_MASTER_BLOCK_ID); + assert applierState.nodes().getMasterNodeId() == null || applierState.metaData().clusterUUIDCommitted(); assert preVoteCollector.getPreVoteResponse().equals(getPreVoteResponse()) : preVoteCollector + " vs " + getPreVoteResponse(); diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java index 53fada396fcef..a9309e9fe638a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java @@ -62,6 +62,7 @@ import java.util.function.BiConsumer; import java.util.function.Function; import java.util.function.LongSupplier; +import java.util.function.Supplier; public class JoinHelper { @@ -84,7 +85,7 @@ public class JoinHelper { final Set> pendingOutgoingJoins = ConcurrentCollections.newConcurrentSet(); public JoinHelper(Settings settings, AllocationService allocationService, MasterService masterService, - TransportService transportService, LongSupplier currentTermSupplier, + TransportService transportService, LongSupplier currentTermSupplier, Supplier currentStateSupplier, BiConsumer joinHandler, Function joinLeaderInTerm, Collection> joinValidators) { this.masterService = masterService; @@ -132,6 +133,13 @@ public ClusterTasksResult execute(ClusterState currentSta transportService.registerRequestHandler(VALIDATE_JOIN_ACTION_NAME, MembershipAction.ValidateJoinRequest::new, ThreadPool.Names.GENERIC, (request, channel, task) -> { + final ClusterState localState = currentStateSupplier.get(); + if (localState.metaData().clusterUUIDCommitted() && + localState.metaData().clusterUUID().equals(request.getState().metaData().clusterUUID()) == false) { + throw new CoordinationStateRejectedException("join validation on cluster state" + + " with a different cluster uuid " + request.getState().metaData().clusterUUID() + + " than local cluster uuid " + localState.metaData().clusterUUID() + ", rejecting"); + } joinValidators.forEach(action -> action.accept(transportService.getLocalNode(), request.getState())); channel.sendResponse(Empty.INSTANCE); }); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index 3cce3f791d2b8..54c3001d9036f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -88,6 +88,7 @@ public class MetaData implements Iterable, Diffable, To private static final Logger logger = LogManager.getLogger(MetaData.class); public static final String ALL = "_all"; + public static final String UNKNOWN_CLUSTER_UUID = "_na_"; public enum XContentContext { /* Custom metadata should be returns as part of API call */ @@ -159,6 +160,7 @@ public interface Custom extends NamedDiffable, ToXContentFragment, Clust private static final NamedDiffableValueSerializer CUSTOM_VALUE_SERIALIZER = new NamedDiffableValueSerializer<>(Custom.class); private final String clusterUUID; + private final boolean clusterUUIDCommitted; private final long version; private final CoordinationMetaData coordinationMetaData; @@ -179,12 +181,13 @@ public interface Custom extends NamedDiffable, ToXContentFragment, Clust private final SortedMap aliasAndIndexLookup; - MetaData(String clusterUUID, long version, CoordinationMetaData coordinationMetaData, + MetaData(String clusterUUID, boolean clusterUUIDCommitted, long version, CoordinationMetaData coordinationMetaData, Settings transientSettings, Settings persistentSettings, ImmutableOpenMap indices, ImmutableOpenMap templates, ImmutableOpenMap customs, String[] allIndices, String[] allOpenIndices, String[] allClosedIndices, SortedMap aliasAndIndexLookup) { this.clusterUUID = clusterUUID; + this.clusterUUIDCommitted = clusterUUIDCommitted; this.version = version; this.coordinationMetaData = coordinationMetaData; this.transientSettings = transientSettings; @@ -218,6 +221,14 @@ public String clusterUUID() { return this.clusterUUID; } + /** + * Whether the current node with the given cluster state is locked into the cluster with the UUID returned by {@link #clusterUUID()}, + * meaning that it will not accept any cluster state with a different clusterUUID. + */ + public boolean clusterUUIDCommitted() { + return this.clusterUUIDCommitted; + } + /** * Returns the merged transient and persistent settings. */ @@ -757,6 +768,12 @@ public static boolean isGlobalStateEquals(MetaData metaData1, MetaData metaData2 if (!metaData1.templates.equals(metaData2.templates())) { return false; } + if (!metaData1.clusterUUID.equals(metaData2.clusterUUID)) { + return false; + } + if (metaData1.clusterUUIDCommitted != metaData2.clusterUUIDCommitted) { + return false; + } // Check if any persistent metadata needs to be saved int customCount1 = 0; for (ObjectObjectCursor cursor : metaData1.customs) { @@ -798,6 +815,7 @@ private static class MetaDataDiff implements Diff { private long version; private String clusterUUID; + private boolean clusterUUIDCommitted; private CoordinationMetaData coordinationMetaData; private Settings transientSettings; private Settings persistentSettings; @@ -807,6 +825,7 @@ private static class MetaDataDiff implements Diff { MetaDataDiff(MetaData before, MetaData after) { clusterUUID = after.clusterUUID; + clusterUUIDCommitted = after.clusterUUIDCommitted; version = after.version; coordinationMetaData = after.coordinationMetaData; transientSettings = after.transientSettings; @@ -818,8 +837,11 @@ private static class MetaDataDiff implements Diff { MetaDataDiff(StreamInput in) throws IOException { clusterUUID = in.readString(); + if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + clusterUUIDCommitted = in.readBoolean(); + } version = in.readLong(); - if (in.getVersion().onOrAfter(Version.V_7_0_0)) { //TODO revisit after Zen2 BWC is implemented + if (in.getVersion().onOrAfter(Version.V_7_0_0)) { coordinationMetaData = new CoordinationMetaData(in); } else { coordinationMetaData = CoordinationMetaData.EMPTY_META_DATA; @@ -836,6 +858,9 @@ private static class MetaDataDiff implements Diff { @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(clusterUUID); + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + out.writeBoolean(clusterUUIDCommitted); + } out.writeLong(version); if (out.getVersion().onOrAfter(Version.V_7_0_0)) { coordinationMetaData.writeTo(out); @@ -851,6 +876,7 @@ public void writeTo(StreamOutput out) throws IOException { public MetaData apply(MetaData part) { Builder builder = builder(); builder.clusterUUID(clusterUUID); + builder.clusterUUIDCommitted(clusterUUIDCommitted); builder.version(version); builder.coordinationMetaData(coordinationMetaData); builder.transientSettings(transientSettings); @@ -866,6 +892,9 @@ public static MetaData readFrom(StreamInput in) throws IOException { Builder builder = new Builder(); builder.version = in.readLong(); builder.clusterUUID = in.readString(); + if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + builder.clusterUUIDCommitted = in.readBoolean(); + } if (in.getVersion().onOrAfter(Version.V_7_0_0)) { builder.coordinationMetaData(new CoordinationMetaData(in)); } @@ -891,6 +920,9 @@ public static MetaData readFrom(StreamInput in) throws IOException { public void writeTo(StreamOutput out) throws IOException { out.writeLong(version); out.writeString(clusterUUID); + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + out.writeBoolean(clusterUUIDCommitted); + } if (out.getVersion().onOrAfter(Version.V_7_0_0)) { coordinationMetaData.writeTo(out); } @@ -930,6 +962,7 @@ public static Builder builder(MetaData metaData) { public static class Builder { private String clusterUUID; + private boolean clusterUUIDCommitted; private long version; private CoordinationMetaData coordinationMetaData = CoordinationMetaData.EMPTY_META_DATA; @@ -941,7 +974,7 @@ public static class Builder { private final ImmutableOpenMap.Builder customs; public Builder() { - clusterUUID = "_na_"; + clusterUUID = UNKNOWN_CLUSTER_UUID; indices = ImmutableOpenMap.builder(); templates = ImmutableOpenMap.builder(); customs = ImmutableOpenMap.builder(); @@ -950,6 +983,7 @@ public Builder() { public Builder(MetaData metaData) { this.clusterUUID = metaData.clusterUUID; + this.clusterUUIDCommitted = metaData.clusterUUIDCommitted; this.coordinationMetaData = metaData.coordinationMetaData; this.transientSettings = metaData.transientSettings; this.persistentSettings = metaData.persistentSettings; @@ -1125,8 +1159,13 @@ public Builder clusterUUID(String clusterUUID) { return this; } + public Builder clusterUUIDCommitted(boolean clusterUUIDCommitted) { + this.clusterUUIDCommitted = clusterUUIDCommitted; + return this; + } + public Builder generateClusterUuidIfNeeded() { - if (clusterUUID.equals("_na_")) { + if (clusterUUID.equals(UNKNOWN_CLUSTER_UUID)) { clusterUUID = UUIDs.randomBase64UUID(); } return this; @@ -1182,8 +1221,9 @@ public MetaData build() { String[] allOpenIndicesArray = allOpenIndices.toArray(new String[allOpenIndices.size()]); String[] allClosedIndicesArray = allClosedIndices.toArray(new String[allClosedIndices.size()]); - return new MetaData(clusterUUID, version, coordinationMetaData, transientSettings, persistentSettings, indices.build(), - templates.build(), customs.build(), allIndicesArray, allOpenIndicesArray, allClosedIndicesArray, aliasAndIndexLookup); + return new MetaData(clusterUUID, clusterUUIDCommitted, version, coordinationMetaData, transientSettings, persistentSettings, + indices.build(), templates.build(), customs.build(), allIndicesArray, allOpenIndicesArray, allClosedIndicesArray, + aliasAndIndexLookup); } private SortedMap buildAliasAndIndexLookup() { @@ -1226,6 +1266,7 @@ public static void toXContent(MetaData metaData, XContentBuilder builder, ToXCon builder.field("version", metaData.version()); builder.field("cluster_uuid", metaData.clusterUUID); + builder.field("cluster_uuid_committed", metaData.clusterUUIDCommitted); builder.startObject("cluster_coordination"); metaData.coordinationMetaData().toXContent(builder, params); @@ -1324,6 +1365,8 @@ public static MetaData fromXContent(XContentParser parser) throws IOException { builder.version = parser.longValue(); } else if ("cluster_uuid".equals(currentFieldName) || "uuid".equals(currentFieldName)) { builder.clusterUUID = parser.text(); + } else if ("cluster_uuid_committed".equals(currentFieldName)) { + builder.clusterUUIDCommitted = parser.booleanValue(); } else { throw new IllegalArgumentException("Unexpected field [" + currentFieldName + "]"); } diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java index 36495914bddec..c3028de1801da 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java @@ -20,6 +20,7 @@ import com.carrotsearch.randomizedtesting.RandomizedContext; import org.apache.logging.log4j.CloseableThreadContext; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -36,6 +37,7 @@ import org.elasticsearch.cluster.coordination.CoordinationState.PersistedState; import org.elasticsearch.cluster.coordination.Coordinator.Mode; import org.elasticsearch.cluster.coordination.CoordinatorTests.Cluster.ClusterNode; +import org.elasticsearch.cluster.metadata.Manifest; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNode.Role; @@ -48,6 +50,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; @@ -59,9 +62,11 @@ import org.elasticsearch.discovery.zen.PublishClusterStateStats; import org.elasticsearch.discovery.zen.UnicastHostsProvider.HostsResolver; import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.gateway.MetaStateService; import org.elasticsearch.gateway.MockGatewayMetaState; import org.elasticsearch.indices.cluster.FakeThreadPoolMasterService; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockLogAppender; import org.elasticsearch.test.disruption.DisruptableMockTransport; import org.elasticsearch.test.disruption.DisruptableMockTransport.ConnectionStatus; import org.elasticsearch.transport.TransportService; @@ -84,6 +89,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.Callable; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.Function; @@ -137,6 +143,13 @@ public class CoordinatorTests extends ESTestCase { private final List nodeEnvironments = new ArrayList<>(); + private final AtomicInteger nextNodeIndex = new AtomicInteger(); + + @Before + public void resetNodeIndexBeforeEachTest() { + nextNodeIndex.set(0); + } + @After public void closeNodeEnvironmentsAfterEachTest() { for (NodeEnvironment nodeEnvironment : nodeEnvironments) { @@ -153,6 +166,7 @@ public void resetPortCounterBeforeEachTest() { // check that runRandomly leads to reproducible results public void testRepeatableTests() throws Exception { final Callable test = () -> { + resetNodeIndexBeforeEachTest(); final Cluster cluster = new Cluster(randomIntBetween(1, 5)); cluster.runRandomly(); final long afterRunRandomly = value(cluster.getAnyNode().getLastAppliedClusterState()); @@ -1001,6 +1015,52 @@ public void testClusterCannotFormWithFailingJoinValidation() { assertTrue(cluster.clusterNodes.stream().allMatch(cn -> cn.getLastAppliedClusterState().version() == 0)); } + public void testCannotJoinClusterWithDifferentUUID() throws IllegalAccessException { + final Cluster cluster1 = new Cluster(randomIntBetween(1, 3)); + cluster1.runRandomly(); + cluster1.stabilise(); + + final Cluster cluster2 = new Cluster(3); + cluster2.runRandomly(); + cluster2.stabilise(); + + final ClusterNode shiftedNode = randomFrom(cluster2.clusterNodes).restartedNode(); + final ClusterNode newNode = cluster1.new ClusterNode(nextNodeIndex.getAndIncrement(), + shiftedNode.getLocalNode(), n -> shiftedNode.persistedState); + cluster1.clusterNodes.add(newNode); + + MockLogAppender mockAppender = new MockLogAppender(); + mockAppender.start(); + mockAppender.addExpectation( + new MockLogAppender.SeenEventExpectation( + "test1", + JoinHelper.class.getCanonicalName(), + Level.INFO, + "*failed to join*")); + Logger joinLogger = LogManager.getLogger(JoinHelper.class); + Loggers.addAppender(joinLogger, mockAppender); + cluster1.runFor(DEFAULT_STABILISATION_TIME, "failing join validation"); + try { + mockAppender.assertAllExpectationsMatched(); + } finally { + Loggers.removeAppender(joinLogger, mockAppender); + mockAppender.stop(); + } + assertTrue(newNode.getLastAppliedClusterState().version() == 0); + + // reset clusterUUIDCommitted (and node / cluster state term) to let node join again + // TODO: use elasticsearch-node detach-cluster tool once it's implemented + final ClusterNode detachedNode = newNode.restartedNode( + metaData -> MetaData.builder(metaData) + .clusterUUIDCommitted(false) + .coordinationMetaData(CoordinationMetaData.builder(metaData.coordinationMetaData()) + .term(0L).build()) + .build(), + term -> 0L); + cluster1.clusterNodes.replaceAll(cn -> cn == newNode ? detachedNode : cn); + cluster1.stabilise(); + } + private static long defaultMillis(Setting setting) { return setting.get(Settings.EMPTY).millis() + Cluster.DEFAULT_DELAY_VARIABILITY; } @@ -1077,7 +1137,8 @@ class Cluster { final Set masterEligibleNodeIds = new HashSet<>(initialNodeCount); clusterNodes = new ArrayList<>(initialNodeCount); for (int i = 0; i < initialNodeCount; i++) { - final ClusterNode clusterNode = new ClusterNode(i, allNodesMasterEligible || i == 0 || randomBoolean()); + final ClusterNode clusterNode = new ClusterNode(nextNodeIndex.getAndIncrement(), + allNodesMasterEligible || i == 0 || randomBoolean()); clusterNodes.add(clusterNode); if (clusterNode.getLocalNode().isMasterNode()) { masterEligibleNodeIds.add(clusterNode.getId()); @@ -1108,10 +1169,9 @@ List addNodesAndStabilise(int newNodesCount) { List addNodes(int newNodesCount) { logger.info("--> adding {} nodes", newNodesCount); - final int nodeSizeAtStart = clusterNodes.size(); final List addedNodes = new ArrayList<>(); for (int i = 0; i < newNodesCount; i++) { - final ClusterNode clusterNode = new ClusterNode(nodeSizeAtStart + i, true); + final ClusterNode clusterNode = new ClusterNode(nextNodeIndex.getAndIncrement(), true); addedNodes.add(clusterNode); } clusterNodes.addAll(addedNodes); @@ -1471,21 +1531,41 @@ class MockPersistedState implements PersistedState { } } - MockPersistedState(DiscoveryNode newLocalNode, MockPersistedState oldState) { + MockPersistedState(DiscoveryNode newLocalNode, MockPersistedState oldState, + Function adaptGlobalMetaData, Function adaptCurrentTerm) { try { if (oldState.nodeEnvironment != null) { nodeEnvironment = oldState.nodeEnvironment; + final MetaStateService metaStateService = new MetaStateService(nodeEnvironment, xContentRegistry()); + final MetaData updatedMetaData = adaptGlobalMetaData.apply(oldState.getLastAcceptedState().metaData()); + if (updatedMetaData != oldState.getLastAcceptedState().metaData()) { + metaStateService.writeGlobalStateAndUpdateManifest("update global state", updatedMetaData); + } + final long updatedTerm = adaptCurrentTerm.apply(oldState.getCurrentTerm()); + if (updatedTerm != oldState.getCurrentTerm()) { + final Manifest manifest = metaStateService.loadManifestOrEmpty(); + metaStateService.writeManifestAndCleanup("update term", + new Manifest(updatedTerm, manifest.getClusterStateVersion(), manifest.getGlobalGeneration(), + manifest.getIndexGenerations())); + } delegate = new MockGatewayMetaState(Settings.EMPTY, nodeEnvironment, xContentRegistry(), newLocalNode) .getPersistedState(Settings.EMPTY, null); } else { nodeEnvironment = null; BytesStreamOutput outStream = new BytesStreamOutput(); outStream.setVersion(Version.CURRENT); - oldState.getLastAcceptedState().writeTo(outStream); + final MetaData updatedMetaData = adaptGlobalMetaData.apply(oldState.getLastAcceptedState().metaData()); + final ClusterState clusterState; + if (updatedMetaData != oldState.getLastAcceptedState().metaData()) { + clusterState = ClusterState.builder(oldState.getLastAcceptedState()).metaData(updatedMetaData).build(); + } else { + clusterState = oldState.getLastAcceptedState(); + } + clusterState.writeTo(outStream); StreamInput inStream = new NamedWriteableAwareStreamInput(outStream.bytes().streamInput(), new NamedWriteableRegistry(ClusterModule.getNamedWriteables())); - delegate = new InMemoryPersistedState(oldState.getCurrentTerm(), ClusterState.readFrom(inStream, - newLocalNode)); // adapts it to new localNode instance + delegate = new InMemoryPersistedState(adaptCurrentTerm.apply(oldState.getCurrentTerm()), + ClusterState.readFrom(inStream, newLocalNode)); // adapts it to new localNode instance } } catch (IOException e) { throw new UncheckedIOException("Unable to create MockPersistedState", e); @@ -1614,12 +1694,17 @@ void close() { } ClusterNode restartedNode() { + return restartedNode(Function.identity(), Function.identity()); + } + + ClusterNode restartedNode(Function adaptGlobalMetaData, Function adaptCurrentTerm) { final TransportAddress address = randomBoolean() ? buildNewFakeTransportAddress() : localNode.getAddress(); final DiscoveryNode newLocalNode = new DiscoveryNode(localNode.getName(), localNode.getId(), UUIDs.randomBase64UUID(random()), // generated deterministically for repeatable tests address.address().getHostString(), address.getAddress(), address, Collections.emptyMap(), localNode.isMasterNode() ? EnumSet.allOf(Role.class) : emptySet(), Version.CURRENT); - return new ClusterNode(nodeIndex, newLocalNode, node -> new MockPersistedState(newLocalNode, persistedState)); + return new ClusterNode(nodeIndex, newLocalNode, + node -> new MockPersistedState(newLocalNode, persistedState, adaptGlobalMetaData, adaptCurrentTerm)); } private PersistedState getPersistedState() { diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/JoinHelperTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/JoinHelperTests.java index ef843717fb469..4361660876c7a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/JoinHelperTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/JoinHelperTests.java @@ -43,7 +43,7 @@ public void testJoinDeduplication() { TransportService transportService = capturingTransport.createTransportService(Settings.EMPTY, deterministicTaskQueue.getThreadPool(), TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> localNode, null, Collections.emptySet()); - JoinHelper joinHelper = new JoinHelper(Settings.EMPTY, null, null, transportService, () -> 0L, + JoinHelper joinHelper = new JoinHelper(Settings.EMPTY, null, null, transportService, () -> 0L, () -> null, (joinRequest, joinCallback) -> { throw new AssertionError(); }, startJoinRequest -> { throw new AssertionError(); }, Collections.emptyList()); transportService.start(); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java index 5dcfccaea5874..685b7cca98a94 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java @@ -411,6 +411,43 @@ public void testXContentWithIndexGraveyard() throws IOException { } } + public void testXContentClusterUUID() throws IOException { + final MetaData originalMeta = MetaData.builder().clusterUUID(UUIDs.randomBase64UUID()) + .clusterUUIDCommitted(randomBoolean()).build(); + final XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject(); + originalMeta.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { + final MetaData fromXContentMeta = MetaData.fromXContent(parser); + assertThat(fromXContentMeta.clusterUUID(), equalTo(originalMeta.clusterUUID())); + assertThat(fromXContentMeta.clusterUUIDCommitted(), equalTo(originalMeta.clusterUUIDCommitted())); + } + } + + public void testSerializationClusterUUID() throws IOException { + final MetaData originalMeta = MetaData.builder().clusterUUID(UUIDs.randomBase64UUID()) + .clusterUUIDCommitted(randomBoolean()).build(); + final BytesStreamOutput out = new BytesStreamOutput(); + originalMeta.writeTo(out); + NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(ClusterModule.getNamedWriteables()); + final MetaData fromStreamMeta = MetaData.readFrom( + new NamedWriteableAwareStreamInput(out.bytes().streamInput(), namedWriteableRegistry) + ); + assertThat(fromStreamMeta.clusterUUID(), equalTo(originalMeta.clusterUUID())); + assertThat(fromStreamMeta.clusterUUIDCommitted(), equalTo(originalMeta.clusterUUIDCommitted())); + } + + public void testMetaDataGlobalStateChangesOnClusterUUIDChanges() { + final MetaData metaData1 = MetaData.builder().clusterUUID(UUIDs.randomBase64UUID()).clusterUUIDCommitted(randomBoolean()).build(); + final MetaData metaData2 = MetaData.builder(metaData1).clusterUUID(UUIDs.randomBase64UUID()).build(); + final MetaData metaData3 = MetaData.builder(metaData1).clusterUUIDCommitted(!metaData1.clusterUUIDCommitted()).build(); + assertFalse(MetaData.isGlobalStateEquals(metaData1, metaData2)); + assertFalse(MetaData.isGlobalStateEquals(metaData1, metaData3)); + final MetaData metaData4 = MetaData.builder(metaData2).clusterUUID(metaData1.clusterUUID()).build(); + assertTrue(MetaData.isGlobalStateEquals(metaData1, metaData4)); + } + private static CoordinationMetaData.VotingConfiguration randomVotingConfig() { return new CoordinationMetaData.VotingConfiguration(Sets.newHashSet(generateRandomStringArray(randomInt(10), 20, false))); } diff --git a/server/src/test/java/org/elasticsearch/discovery/ClusterDisruptionIT.java b/server/src/test/java/org/elasticsearch/discovery/ClusterDisruptionIT.java index d94c34c7b33eb..330c73b9c02c5 100644 --- a/server/src/test/java/org/elasticsearch/discovery/ClusterDisruptionIT.java +++ b/server/src/test/java/org/elasticsearch/discovery/ClusterDisruptionIT.java @@ -28,6 +28,7 @@ import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.action.shard.ShardStateAction; +import org.elasticsearch.cluster.coordination.ClusterBootstrapService; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.routing.Murmur3HashFunction; import org.elasticsearch.cluster.routing.ShardRouting; @@ -377,6 +378,33 @@ public boolean clearData(String nodeName) { assertTrue(client().prepareGet("index", "_doc", "1").get().isExists()); } + public void testCannotJoinIfMasterLostDataFolder() throws Exception { + String masterNode = internalCluster().startMasterOnlyNode(); + String dataNode = internalCluster().startDataOnlyNode(); + + internalCluster().restartNode(masterNode, new InternalTestCluster.RestartCallback() { + @Override + public boolean clearData(String nodeName) { + return true; + } + + @Override + public Settings onNodeStopped(String nodeName) { + return Settings.builder().put(ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.getKey(), nodeName).build(); + } + + @Override + public boolean validateClusterForming() { + return false; + } + }); + + assertFalse(internalCluster().client(masterNode).admin().cluster().prepareHealth().get().isTimedOut()); + assertTrue(internalCluster().client(masterNode).admin().cluster().prepareHealth().setWaitForNodes("2").setTimeout("2s").get() + .isTimedOut()); + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(dataNode)); // otherwise we will fail during clean-up + } + /** * Tests that indices are properly deleted even if there is a master transition in between. * Test for https://github.com/elastic/elasticsearch/issues/11665 diff --git a/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java b/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java index b34bcf87bdbd8..cae33db90a6bc 100644 --- a/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java @@ -263,12 +263,12 @@ public void testMixCurrentAndRecoveredState() { .blocks(ClusterBlocks.builder().addGlobalBlock(CLUSTER_READ_ONLY_BLOCK).build()) .metaData(metaData) .build(); - assertThat(recoveredState.metaData().clusterUUID(), equalTo("_na_")); + assertThat(recoveredState.metaData().clusterUUID(), equalTo(MetaData.UNKNOWN_CLUSTER_UUID)); final ClusterState updatedState = mixCurrentStateAndRecoveredState(currentState, recoveredState); - assertThat(updatedState.metaData().clusterUUID(), not(equalTo("_na_"))); - assertTrue(MetaData.isGlobalStateEquals(metaData, updatedState.metaData())); + assertThat(updatedState.metaData().clusterUUID(), not(equalTo(MetaData.UNKNOWN_CLUSTER_UUID))); + assertFalse(MetaData.isGlobalStateEquals(metaData, updatedState.metaData())); assertThat(updatedState.metaData().index("test"), equalTo(indexMetaData)); assertTrue(updatedState.blocks().hasGlobalBlock(STATE_NOT_RECOVERED_BLOCK)); assertTrue(updatedState.blocks().hasGlobalBlock(CLUSTER_READ_ONLY_BLOCK)); diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStatePersistedStateTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStatePersistedStateTests.java index 921bcac3d4c65..8ccfa5e406ae2 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStatePersistedStateTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStatePersistedStateTests.java @@ -213,22 +213,24 @@ public void testMarkAcceptedConfigAsCommitted() throws IOException { } while (coordinationMetaData.getLastAcceptedConfiguration().equals(coordinationMetaData.getLastCommittedConfiguration())); ClusterState state = createClusterState(randomNonNegativeLong(), - MetaData.builder().coordinationMetaData(coordinationMetaData).build()); + MetaData.builder().coordinationMetaData(coordinationMetaData) + .clusterUUID(randomAlphaOfLength(10)).build()); gateway.setLastAcceptedState(state); gateway = maybeNew(gateway); assertThat(gateway.getLastAcceptedState().getLastAcceptedConfiguration(), not(equalTo(gateway.getLastAcceptedState().getLastCommittedConfiguration()))); - gateway.markLastAcceptedConfigAsCommitted(); + gateway.markLastAcceptedStateAsCommitted(); CoordinationMetaData expectedCoordinationMetaData = CoordinationMetaData.builder(coordinationMetaData) .lastCommittedConfiguration(coordinationMetaData.getLastAcceptedConfiguration()).build(); ClusterState expectedClusterState = - ClusterState.builder(state).metaData(MetaData.builder().coordinationMetaData(expectedCoordinationMetaData).build()).build(); + ClusterState.builder(state).metaData(MetaData.builder().coordinationMetaData(expectedCoordinationMetaData) + .clusterUUID(state.metaData().clusterUUID()).clusterUUIDCommitted(true).build()).build(); gateway = maybeNew(gateway); assertClusterStateEqual(expectedClusterState, gateway.getLastAcceptedState()); - gateway.markLastAcceptedConfigAsCommitted(); + gateway.markLastAcceptedStateAsCommitted(); gateway = maybeNew(gateway); assertClusterStateEqual(expectedClusterState, gateway.getLastAcceptedState()); From 6d1693ff49974b1aef9741551df33ad3e851cec4 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Tue, 29 Jan 2019 15:09:40 +0000 Subject: [PATCH 016/100] [ML] Prevent submit after autodetect worker is stopped (#37700) Runnables can be submitted to AutodetectProcessManager.AutodetectWorkerExecutorService without error after it has been shutdown which can lead to requests timing out as their handlers are never called by the terminated executor. This change throws an EsRejectedExecutionException if a runnable is submitted after after the shutdown and calls AbstractRunnable.onRejection on any tasks not run. Closes #37108 --- .../autodetect/AutodetectProcessManager.java | 27 ++++++- .../AutodetectProcessManagerTests.java | 70 +++++++++++++++++++ 2 files changed, 96 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java index 21aa08e14f217..1d8f4f273601f 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java @@ -77,6 +77,7 @@ import java.nio.file.Path; import java.time.Duration; import java.time.ZonedDateTime; +import java.util.ArrayList; import java.util.Date; import java.util.Iterator; import java.util.List; @@ -831,7 +832,16 @@ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedE } @Override - public void execute(Runnable command) { + public synchronized void execute(Runnable command) { + if (isShutdown()) { + EsRejectedExecutionException rejected = new EsRejectedExecutionException("autodetect worker service has shutdown", true); + if (command instanceof AbstractRunnable) { + ((AbstractRunnable) command).onRejection(rejected); + } else { + throw rejected; + } + } + boolean added = queue.offer(contextHolder.preserveContext(command)); if (added == false) { throw new ElasticsearchStatusException("Unable to submit operation", RestStatus.TOO_MANY_REQUESTS); @@ -851,6 +861,21 @@ void start() { EsExecutors.rethrowErrors(contextHolder.unwrap(runnable)); } } + + synchronized (this) { + // if shutdown with tasks pending notify the handlers + if (queue.isEmpty() == false) { + List notExecuted = new ArrayList<>(); + queue.drainTo(notExecuted); + + for (Runnable runnable : notExecuted) { + if (runnable instanceof AbstractRunnable) { + ((AbstractRunnable) runnable).onRejection( + new EsRejectedExecutionException("unable to process as autodetect worker service has shutdown", true)); + } + } + } + } } catch (InterruptedException e) { Thread.currentThread().interrupt(); } finally { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java index 82788d4500b09..6b4fd270b1bb7 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.util.concurrent.ThreadContext; @@ -54,6 +55,7 @@ import org.elasticsearch.xpack.ml.job.process.autodetect.params.TimeRange; import org.elasticsearch.xpack.ml.job.process.normalizer.NormalizerFactory; import org.elasticsearch.xpack.ml.notifications.Auditor; +import org.junit.After; import org.junit.Before; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; @@ -72,9 +74,11 @@ import java.util.TreeMap; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; @@ -125,6 +129,8 @@ public class AutodetectProcessManagerTests extends ESTestCase { private Quantiles quantiles = new Quantiles("foo", new Date(), "state"); private Set filters = new HashSet<>(); + private ThreadPool threadPool; + @Before public void setup() throws Exception { Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build(); @@ -159,8 +165,16 @@ public void setup() throws Exception { handler.accept(buildAutodetectParams()); return null; }).when(jobResultsProvider).getAutodetectParams(any(), any(), any()); + + threadPool = new TestThreadPool("AutodetectProcessManagerTests"); } + @After + public void stopThreadPool() throws InterruptedException { + terminate(threadPool); + } + + public void testMaxOpenJobsSetting_givenDefault() { int maxOpenJobs = AutodetectProcessManager.MAX_OPEN_JOBS_PER_NODE.get(Settings.EMPTY); assertEquals(20, maxOpenJobs); @@ -690,6 +704,62 @@ public void testAutodetectWorkerExecutorServiceDoesNotSwallowErrors() { } } + public void testAutodetectWorkerExecutorService_SubmitAfterShutdown() { + AutodetectProcessManager.AutodetectWorkerExecutorService executor = + new AutodetectWorkerExecutorService(new ThreadContext(Settings.EMPTY)); + + threadPool.generic().execute(() -> executor.start()); + executor.shutdown(); + expectThrows(EsRejectedExecutionException.class, () -> executor.execute(() -> {})); + } + + public void testAutodetectWorkerExecutorService_TasksNotExecutedCallHandlerOnShutdown() + throws InterruptedException, ExecutionException { + AutodetectProcessManager.AutodetectWorkerExecutorService executor = + new AutodetectWorkerExecutorService(new ThreadContext(Settings.EMPTY)); + + CountDownLatch latch = new CountDownLatch(1); + + Future executorFinished = threadPool.generic().submit(() -> executor.start()); + + // run a task that will block while the others are queued up + executor.execute(() -> { + try { + latch.await(); + } catch (InterruptedException e) { + } + }); + + AtomicBoolean runnableShouldNotBeCalled = new AtomicBoolean(false); + executor.execute(() -> runnableShouldNotBeCalled.set(true)); + + AtomicInteger onFailureCallCount = new AtomicInteger(); + AtomicInteger doRunCallCount = new AtomicInteger(); + for (int i=0; i<2; i++) { + executor.execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + onFailureCallCount.incrementAndGet(); + } + + @Override + protected void doRun() { + doRunCallCount.incrementAndGet(); + } + }); + } + + // now shutdown + executor.shutdown(); + latch.countDown(); + executorFinished.get(); + + assertFalse(runnableShouldNotBeCalled.get()); + // the AbstractRunnables should have had their callbacks called + assertEquals(2, onFailureCallCount.get()); + assertEquals(0, doRunCallCount.get()); + } + private AutodetectProcessManager createNonSpyManager(String jobId) { Client client = mock(Client.class); ThreadPool threadPool = mock(ThreadPool.class); From 218df3009a8d3224c416f9ab05893e637714e366 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Tue, 29 Jan 2019 10:23:05 -0500 Subject: [PATCH 017/100] Move update and delete by query to use seq# for optimistic concurrency control (#37857) The delete and update by query APIs both offer protection against overriding concurrent user changes to the documents they touch. They currently are using internal versioning. This PR changes that to rely on sequences numbers and primary terms. Relates #37639 Relates #36148 Relates #10708 --- docs/reference/docs/index_.asciidoc | 8 --- .../AbstractAsyncBulkByScrollAction.java | 26 ++++---- .../reindex/AsyncDeleteByQueryAction.java | 26 +++++--- .../index/reindex/TransportReindexAction.java | 17 ++--- .../reindex/TransportUpdateByQueryAction.java | 30 +++++---- .../reindex/remote/RemoteResponseParsers.java | 12 ++-- .../reindex/AsyncBulkByScrollActionTests.java | 9 +-- .../reindex/UpdateByQueryMetadataTests.java | 12 ++-- .../reindex/UpdateByQueryWithScriptTests.java | 3 +- .../test/delete_by_query/10_basic.yml | 64 ++++++++++++++++++- .../test/delete_by_query/40_versioning.yml | 4 ++ .../test/update_by_query/10_basic.yml | 49 +++++++++++++- .../test/update_by_query/40_versioning.yml | 3 + .../reindex/ClientScrollableHitSource.java | 10 +++ .../index/reindex/ScrollableHitSource.java | 32 ++++++++++ 15 files changed, 230 insertions(+), 75 deletions(-) diff --git a/docs/reference/docs/index_.asciidoc b/docs/reference/docs/index_.asciidoc index 257b88289d87a..e8a681567d622 100644 --- a/docs/reference/docs/index_.asciidoc +++ b/docs/reference/docs/index_.asciidoc @@ -372,14 +372,6 @@ the current document version of 1. If the document was already updated and its version was set to 2 or higher, the indexing command will fail and result in a conflict (409 http status code). -WARNING: External versioning supports the value 0 as a valid version number. -This allows the version to be in sync with an external versioning system -where version numbers start from zero instead of one. It has the side effect -that documents with version number equal to zero can neither be updated -using the <> nor be deleted -using the <> as long as their -version number is equal to zero. - A nice side effect is that there is no need to maintain strict ordering of async indexing operations executed as a result of changes to a source database, as long as version numbers from the source database are used. diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java index e55dab1c38f64..9617d2a3774da 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java @@ -35,7 +35,6 @@ import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.client.ParentTaskAssigningClient; -import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AbstractRunnable; @@ -50,6 +49,7 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.UpdateScript; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.threadpool.ThreadPool; @@ -88,7 +88,6 @@ public abstract class AbstractAsyncBulkByScrollActionrequest variables all representing child @@ -111,9 +110,10 @@ public abstract class AbstractAsyncBulkByScrollAction, ScrollableHitSource.Hit, RequestWrapper> scriptApplier; - public AbstractAsyncBulkByScrollAction(BulkByScrollTask task, Logger logger, ParentTaskAssigningClient client, - ThreadPool threadPool, Request mainRequest, ScriptService scriptService, ClusterState clusterState, - ActionListener listener) { + public AbstractAsyncBulkByScrollAction(BulkByScrollTask task, boolean needsSourceDocumentVersions, + boolean needsSourceDocumentSeqNoAndPrimaryTerm, Logger logger, ParentTaskAssigningClient client, + ThreadPool threadPool, Request mainRequest, ScriptService scriptService, + ActionListener listener) { this.task = task; if (!task.isWorker()) { @@ -125,7 +125,6 @@ public AbstractAsyncBulkByScrollAction(BulkByScrollTask task, Logger logger, Par this.client = client; this.threadPool = threadPool; this.scriptService = scriptService; - this.clusterState = clusterState; this.mainRequest = mainRequest; this.listener = listener; BackoffPolicy backoffPolicy = buildBackoffPolicy(); @@ -137,11 +136,13 @@ public AbstractAsyncBulkByScrollAction(BulkByScrollTask task, Logger logger, Par * them and if we add _doc as the first sort by default then sorts will never work.... So we add it here, only if there isn't * another sort. */ - List> sorts = mainRequest.getSearchRequest().source().sorts(); + final SearchSourceBuilder sourceBuilder = mainRequest.getSearchRequest().source(); + List> sorts = sourceBuilder.sorts(); if (sorts == null || sorts.isEmpty()) { - mainRequest.getSearchRequest().source().sort(fieldSort("_doc")); + sourceBuilder.sort(fieldSort("_doc")); } - mainRequest.getSearchRequest().source().version(needsSourceDocumentVersions()); + sourceBuilder.version(needsSourceDocumentVersions); + sourceBuilder.seqNoAndPrimaryTerm(needsSourceDocumentSeqNoAndPrimaryTerm); } /** @@ -153,12 +154,7 @@ public BiFunction, ScrollableHitSource.Hit, RequestWrapper> // The default script applier executes a no-op return (request, searchHit) -> request; } - - /** - * Does this operation need the versions of the source documents? - */ - protected abstract boolean needsSourceDocumentVersions(); - + /** * Build the {@link RequestWrapper} for a single search hit. This shouldn't handle * metadata or scripting. That will be handled by copyMetadata and diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AsyncDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AsyncDeleteByQueryAction.java index c86911649ac34..5026ec0f79c57 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AsyncDeleteByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AsyncDeleteByQueryAction.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.reindex; import org.apache.logging.log4j.Logger; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.client.ParentTaskAssigningClient; @@ -31,20 +32,20 @@ * Implementation of delete-by-query using scrolling and bulk. */ public class AsyncDeleteByQueryAction extends AbstractAsyncBulkByScrollAction { + private final boolean useSeqNoForCAS; + public AsyncDeleteByQueryAction(BulkByScrollTask task, Logger logger, ParentTaskAssigningClient client, ThreadPool threadPool, DeleteByQueryRequest request, ScriptService scriptService, ClusterState clusterState, ActionListener listener) { - super(task, logger, client, threadPool, request, scriptService, clusterState, listener); + super(task, + // not all nodes support sequence number powered optimistic concurrency control, we fall back to version + clusterState.nodes().getMinNodeVersion().onOrAfter(Version.V_6_7_0) == false, + // all nodes support sequence number powered optimistic concurrency control and we can use it + clusterState.nodes().getMinNodeVersion().onOrAfter(Version.V_6_7_0), + logger, client, threadPool, request, scriptService, listener); + useSeqNoForCAS = clusterState.nodes().getMinNodeVersion().onOrAfter(Version.V_6_7_0); } - @Override - protected boolean needsSourceDocumentVersions() { - /* - * We always need the version of the source document so we can report a version conflict if we try to delete it and it has been - * changed. - */ - return true; - } @Override protected boolean accept(ScrollableHitSource.Hit doc) { @@ -59,7 +60,12 @@ protected RequestWrapper buildRequest(ScrollableHitSource.Hit doc delete.index(doc.getIndex()); delete.type(doc.getType()); delete.id(doc.getId()); - delete.version(doc.getVersion()); + if (useSeqNoForCAS) { + delete.setIfSeqNo(doc.getSeqNo()); + delete.setIfPrimaryTerm(doc.getPrimaryTerm()); + } else { + delete.version(doc.getVersion()); + } return wrap(delete); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java index 0acc9a7b37dbc..3073b17d5bf95 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java @@ -259,16 +259,13 @@ static class AsyncIndexBySearchAction extends AbstractAsyncBulkByScrollAction listener) { - super(task, logger, client, threadPool, request, scriptService, clusterState, listener); - } - - @Override - protected boolean needsSourceDocumentVersions() { - /* - * We only need the source version if we're going to use it when write and we only do that when the destination request uses - * external versioning. - */ - return mainRequest.getDestination().versionType() != VersionType.INTERNAL; + super(task, + /* + * We only need the source version if we're going to use it when write and we only do that when the destination request uses + * external versioning. + */ + request.getDestination().versionType() != VersionType.INTERNAL, + false, logger, client, threadPool, request, scriptService, listener); } @Override diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java index fc86583832619..9ed7744f8a27a 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.reindex; import org.apache.logging.log4j.Logger; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.ActionFilters; @@ -81,19 +82,19 @@ protected void doExecute(Task task, UpdateByQueryRequest request, ActionListener * Simple implementation of update-by-query using scrolling and bulk. */ static class AsyncIndexBySearchAction extends AbstractAsyncBulkByScrollAction { + + private final boolean useSeqNoForCAS; + AsyncIndexBySearchAction(BulkByScrollTask task, Logger logger, ParentTaskAssigningClient client, ThreadPool threadPool, UpdateByQueryRequest request, ScriptService scriptService, ClusterState clusterState, ActionListener listener) { - super(task, logger, client, threadPool, request, scriptService, clusterState, listener); - } - - @Override - protected boolean needsSourceDocumentVersions() { - /* - * We always need the version of the source document so we can report a version conflict if we try to delete it and it has - * been changed. - */ - return true; + super(task, + // not all nodes support sequence number powered optimistic concurrency control, we fall back to version + clusterState.nodes().getMinNodeVersion().onOrAfter(Version.V_6_7_0) == false, + // all nodes support sequence number powered optimistic concurrency control and we can use it + clusterState.nodes().getMinNodeVersion().onOrAfter(Version.V_6_7_0), + logger, client, threadPool, request, scriptService, listener); + useSeqNoForCAS = clusterState.nodes().getMinNodeVersion().onOrAfter(Version.V_6_7_0); } @Override @@ -112,8 +113,13 @@ protected RequestWrapper buildRequest(ScrollableHitSource.Hit doc) index.type(doc.getType()); index.id(doc.getId()); index.source(doc.getSource(), doc.getXContentType()); - index.versionType(VersionType.INTERNAL); - index.version(doc.getVersion()); + if (useSeqNoForCAS) { + index.setIfSeqNo(doc.getSeqNo()); + index.setIfPrimaryTerm(doc.getPrimaryTerm()); + } else { + index.versionType(VersionType.INTERNAL); + index.version(doc.getVersion()); + } index.setPipeline(mainRequest.getPipeline()); return wrap(index); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java index c7e814237d6b9..ccd177b9174ab 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java @@ -21,13 +21,9 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.Version; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.index.reindex.ScrollableHitSource.BasicHit; -import org.elasticsearch.index.reindex.ScrollableHitSource.Hit; -import org.elasticsearch.index.reindex.ScrollableHitSource.Response; -import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.xcontent.ConstructingObjectParser; @@ -37,6 +33,10 @@ import org.elasticsearch.common.xcontent.XContentLocation; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.reindex.ScrollableHitSource.BasicHit; +import org.elasticsearch.index.reindex.ScrollableHitSource.Hit; +import org.elasticsearch.index.reindex.ScrollableHitSource.Response; +import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure; import org.elasticsearch.search.SearchHits; import java.io.IOException; @@ -90,6 +90,8 @@ private RemoteResponseParsers() {} ParseField routingField = new ParseField("_routing"); ParseField ttlField = new ParseField("_ttl"); ParseField parentField = new ParseField("_parent"); + ParseField seqNoField = new ParseField("_seq_no"); + ParseField primaryTermField = new ParseField("_primary_term"); HIT_PARSER.declareString(BasicHit::setRouting, routingField); // Pre-2.0.0 routing come back in "fields" class Fields { diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java index e6aee3596b1bd..cd23bf03add59 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java @@ -677,13 +677,8 @@ private void simulateScrollResponse(DummyAsyncBulkByScrollAction action, TimeVal private class DummyAsyncBulkByScrollAction extends AbstractAsyncBulkByScrollAction { DummyAsyncBulkByScrollAction() { - super(testTask, AsyncBulkByScrollActionTests.this.logger, new ParentTaskAssigningClient(client, localNode, testTask), - client.threadPool(), testRequest, null, null, listener); - } - - @Override - protected boolean needsSourceDocumentVersions() { - return randomBoolean(); + super(testTask, randomBoolean(), randomBoolean(), AsyncBulkByScrollActionTests.this.logger, + new ParentTaskAssigningClient(client, localNode, testTask), client.threadPool(), testRequest, null, listener); } @Override diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryMetadataTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryMetadataTests.java index 3ce8884ff92fb..95ee787f13f63 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryMetadataTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryMetadataTests.java @@ -19,12 +19,14 @@ package org.elasticsearch.index.reindex; -import org.elasticsearch.index.reindex.ScrollableHitSource.Hit; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.index.reindex.ScrollableHitSource.Hit; public class UpdateByQueryMetadataTests - extends AbstractAsyncBulkByScrollActionMetadataTestCase { - public void testRoutingIsCopied() throws Exception { + extends AbstractAsyncBulkByScrollActionMetadataTestCase { + + public void testRoutingIsCopied() { IndexRequest index = new IndexRequest(); action().copyMetadata(AbstractAsyncBulkByScrollAction.wrap(index), doc().setRouting("foo")); assertEquals("foo", index.routing()); @@ -43,12 +45,12 @@ protected UpdateByQueryRequest request() { private class TestAction extends TransportUpdateByQueryAction.AsyncIndexBySearchAction { TestAction() { super(UpdateByQueryMetadataTests.this.task, UpdateByQueryMetadataTests.this.logger, null, - UpdateByQueryMetadataTests.this.threadPool, request(), null, null, listener()); + UpdateByQueryMetadataTests.this.threadPool, request(), null, ClusterState.EMPTY_STATE, listener()); } @Override public AbstractAsyncBulkByScrollAction.RequestWrapper copyMetadata(AbstractAsyncBulkByScrollAction.RequestWrapper request, - Hit doc) { + Hit doc) { return super.copyMetadata(request, doc); } } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java index 90b78b9e1080d..0eb2a1cfb7d0a 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.reindex; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.script.ScriptService; import java.util.Date; @@ -54,6 +55,6 @@ protected UpdateByQueryRequest request() { @Override protected TransportUpdateByQueryAction.AsyncIndexBySearchAction action(ScriptService scriptService, UpdateByQueryRequest request) { return new TransportUpdateByQueryAction.AsyncIndexBySearchAction(task, logger, null, threadPool, request, scriptService, - null, listener()); + ClusterState.EMPTY_STATE, listener()); } } diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yml index 9f9a99f0f5c2e..7ce6b86c6b525 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yml @@ -89,7 +89,11 @@ - is_false: response.task --- -"Response for version conflict": +"Response for version conflict (version powered)": + - skip: + version: "6.7.0 - " + reason: reindex moved to rely on sequence numbers for concurrency control + - do: indices.create: index: test @@ -143,6 +147,64 @@ - match: {count: 1} +--- +"Response for version conflict (seq no powered)": + - skip: + version: " - 6.6.99" + reason: reindex moved to rely on sequence numbers for concurrency control + + - do: + indices.create: + index: test + body: + settings: + index.refresh_interval: -1 + - do: + index: + index: test + type: _doc + id: 1 + body: { "text": "test" } + - do: + indices.refresh: {} + # Creates a new version for reindex to miss on scan. + - do: + index: + index: test + type: _doc + id: 1 + body: { "text": "test2" } + + - do: + catch: conflict + delete_by_query: + index: test + body: + query: + match_all: {} + + - match: {deleted: 0} + - match: {version_conflicts: 1} + - match: {batches: 1} + - match: {failures.0.index: test} + - match: {failures.0.type: _doc} + - match: {failures.0.id: "1"} + - match: {failures.0.status: 409} + - match: {failures.0.cause.type: version_conflict_engine_exception} + - match: {failures.0.cause.reason: "/\\[_doc\\]\\[1\\]:.version.conflict,.required.seqNo.\\[\\d+\\]/"} + - match: {failures.0.cause.shard: /\d+/} + - match: {failures.0.cause.index: test} + - gte: { took: 0 } + + - do: + indices.refresh: {} + + - do: + count: + index: test + + - match: {count: 1} + --- "Response for version conflict with conflicts=proceed": - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/40_versioning.yml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/40_versioning.yml index 1e2afacca148e..8448b229b258b 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/40_versioning.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/40_versioning.yml @@ -1,5 +1,9 @@ --- "delete_by_query fails to delete documents with version number equal to zero": + - skip: + version: "6.7.0 - " + reason: reindex moved to rely on sequence numbers for concurrency control + - do: index: index: index1 diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yml index d3881035d924f..eb76681c0b0d1 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yml @@ -74,7 +74,10 @@ - is_false: response.deleted --- -"Response for version conflict": +"Response for version conflict (version powered)": + - skip: + version: "6.7.0 - " + reason: reindex moved to rely on sequence numbers for concurrency control - do: indices.create: index: test @@ -115,6 +118,50 @@ - match: {failures.0.cause.index: test} - gte: { took: 0 } +--- +"Response for version conflict (seq no powered)": + - skip: + version: " - 6.6.99" + reason: reindex moved to rely on sequence numbers for concurrency control + - do: + indices.create: + index: test + body: + settings: + index.refresh_interval: -1 + - do: + index: + index: test + type: _doc + id: 1 + body: { "text": "test" } + - do: + indices.refresh: {} + # Creates a new version for reindex to miss on scan. + - do: + index: + index: test + type: _doc + id: 1 + body: { "text": "test2" } + + - do: + catch: conflict + update_by_query: + index: test + - match: {updated: 0} + - match: {version_conflicts: 1} + - match: {batches: 1} + - match: {failures.0.index: test} + - match: {failures.0.type: _doc} + - match: {failures.0.id: "1"} + - match: {failures.0.status: 409} + - match: {failures.0.cause.type: version_conflict_engine_exception} + - match: {failures.0.cause.reason: "/\\[_doc\\]\\[1\\]:.version.conflict,.required.seqNo.\\[\\d+\\]/"} + - match: {failures.0.cause.shard: /\d+/} + - match: {failures.0.cause.index: test} + - gte: { took: 0 } + --- "Response for version conflict with conflicts=proceed": - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/40_versioning.yml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/40_versioning.yml index 7d2083f925b99..d14691be53b83 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/40_versioning.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/40_versioning.yml @@ -24,6 +24,9 @@ --- "update_by_query fails to update documents with version number equal to zero": + - skip: + version: "6.7.0 - " + reason: reindex moved to rely on sequence numbers for concurrency control - do: index: index: index1 diff --git a/server/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java b/server/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java index 7ba3013497990..4fc5770709c1b 100644 --- a/server/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java +++ b/server/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java @@ -241,6 +241,16 @@ public long getVersion() { return delegate.getVersion(); } + @Override + public long getSeqNo() { + return delegate.getSeqNo(); + } + + @Override + public long getPrimaryTerm() { + return delegate.getPrimaryTerm(); + } + @Override public String getRouting() { return fieldValue(RoutingFieldMapper.NAME); diff --git a/server/src/main/java/org/elasticsearch/index/reindex/ScrollableHitSource.java b/server/src/main/java/org/elasticsearch/index/reindex/ScrollableHitSource.java index a3901bb7a568b..dc8d69ff4f0d1 100644 --- a/server/src/main/java/org/elasticsearch/index/reindex/ScrollableHitSource.java +++ b/server/src/main/java/org/elasticsearch/index/reindex/ScrollableHitSource.java @@ -33,6 +33,7 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.search.SearchHit; import org.elasticsearch.threadpool.ThreadPool; @@ -190,6 +191,17 @@ public interface Hit { * internal APIs. */ long getVersion(); + + /** + * The sequence number of the match or {@link SequenceNumbers#UNASSIGNED_SEQ_NO} if sequence numbers weren't requested. + */ + long getSeqNo(); + + /** + * The primary term of the match or {@link SequenceNumbers#UNASSIGNED_PRIMARY_TERM} if sequence numbers weren't requested. + */ + long getPrimaryTerm(); + /** * The source of the hit. Returns null if the source didn't come back from the search, usually because it source wasn't stored at * all. @@ -217,6 +229,8 @@ public static class BasicHit implements Hit { private BytesReference source; private XContentType xContentType; private String routing; + private long seqNo; + private long primaryTerm; public BasicHit(String index, String type, String id, long version) { this.index = index; @@ -245,6 +259,16 @@ public long getVersion() { return version; } + @Override + public long getSeqNo() { + return seqNo; + } + + @Override + public long getPrimaryTerm() { + return primaryTerm; + } + @Override public BytesReference getSource() { return source; @@ -270,6 +294,14 @@ public BasicHit setRouting(String routing) { this.routing = routing; return this; } + + public void setSeqNo(long seqNo) { + this.seqNo = seqNo; + } + + public void setPrimaryTerm(long primaryTerm) { + this.primaryTerm = primaryTerm; + } } /** From 5f106a27eaaf6deded97b421230065aa3b487409 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Tue, 29 Jan 2019 15:41:35 +0000 Subject: [PATCH 018/100] [ML] Add meta information to all ML indices (#37964) This change adds a _meta field storing the version in which the index mappings were last updated to the 3 ML indices that didn't previously have one: - .ml-annotations - .ml-meta - .ml-notifications All other ML indices already had such a _meta field. This field will be useful if we ever need to automatically update the index mappings during a future upgrade. --- .../org/elasticsearch/xpack/core/ml/MlMetaIndex.java | 1 + .../xpack/core/ml/annotations/AnnotationIndex.java | 8 +++++--- .../core/ml/job/persistence/ElasticsearchMappings.java | 9 +++++---- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java index 9014c415f16bb..b1ec651500e0f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java @@ -29,6 +29,7 @@ public static XContentBuilder docMapping() throws IOException { XContentBuilder builder = jsonBuilder(); builder.startObject(); builder.startObject(TYPE); + ElasticsearchMappings.addMetaInformation(builder); ElasticsearchMappings.addDefaultMapping(builder); builder.startObject(ElasticsearchMappings.PROPERTIES) .startObject(Calendar.ID.getPreferredName()) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java index 843be6596c380..437aa40c925f2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java @@ -109,10 +109,11 @@ public static void createAnnotationsIndexIfNecessary(Settings settings, Client c } public static XContentBuilder annotationsMapping() throws IOException { - return jsonBuilder() + XContentBuilder builder = jsonBuilder() .startObject() - .startObject(ElasticsearchMappings.DOC_TYPE) - .startObject(ElasticsearchMappings.PROPERTIES) + .startObject(ElasticsearchMappings.DOC_TYPE); + ElasticsearchMappings.addMetaInformation(builder); + builder.startObject(ElasticsearchMappings.PROPERTIES) .startObject(Annotation.ANNOTATION.getPreferredName()) .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.TEXT) .endObject() @@ -143,5 +144,6 @@ public static XContentBuilder annotationsMapping() throws IOException { .endObject() .endObject() .endObject(); + return builder; } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java index 0eb2e666916dc..d51a8f10e4a5a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java @@ -960,10 +960,10 @@ private static void addModelSizeStatsFieldsToMapping(XContentBuilder builder) th } public static XContentBuilder auditMessageMapping() throws IOException { - return jsonBuilder() - .startObject() - .startObject(AuditMessage.TYPE.getPreferredName()) - .startObject(PROPERTIES) + XContentBuilder builder = jsonBuilder().startObject() + .startObject(AuditMessage.TYPE.getPreferredName()); + addMetaInformation(builder); + builder.startObject(PROPERTIES) .startObject(Job.ID.getPreferredName()) .field(TYPE, KEYWORD) .endObject() @@ -987,6 +987,7 @@ public static XContentBuilder auditMessageMapping() throws IOException { .endObject() .endObject() .endObject(); + return builder; } static String[] mappingRequiresUpdate(ClusterState state, String[] concreteIndices, Version minVersion) throws IOException { From 1579ac032b690d98817f38be6911e758256bd08f Mon Sep 17 00:00:00 2001 From: markharwood Date: Tue, 29 Jan 2019 16:43:24 +0000 Subject: [PATCH 019/100] Added missing eclipse-build.gradle files (#37980) Eclipse build files were missing so .eclipse project files were not being generated. Closes #37973 --- libs/geo/src/main/eclipse-build.gradle | 3 +++ libs/geo/src/test/eclipse-build.gradle | 6 ++++++ 2 files changed, 9 insertions(+) create mode 100644 libs/geo/src/main/eclipse-build.gradle create mode 100644 libs/geo/src/test/eclipse-build.gradle diff --git a/libs/geo/src/main/eclipse-build.gradle b/libs/geo/src/main/eclipse-build.gradle new file mode 100644 index 0000000000000..4864452846b13 --- /dev/null +++ b/libs/geo/src/main/eclipse-build.gradle @@ -0,0 +1,3 @@ + +// this is just shell gradle file for eclipse to have separate projects for geo src and tests +apply from: '../../build.gradle' diff --git a/libs/geo/src/test/eclipse-build.gradle b/libs/geo/src/test/eclipse-build.gradle new file mode 100644 index 0000000000000..8dc16debdde32 --- /dev/null +++ b/libs/geo/src/test/eclipse-build.gradle @@ -0,0 +1,6 @@ + +// this is just shell gradle file for eclipse to have separate projects for geo src and tests +apply from: '../../build.gradle' +dependencies { + testCompile project(':libs:elasticsearch-geo') +} \ No newline at end of file From d05a4b9d14bbcb0f8900d871478d3c97b72c5ba1 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Tue, 29 Jan 2019 18:56:20 +0200 Subject: [PATCH 020/100] Get Aliases with wildcard exclusion expression (#34230) This commit adds the code in the HTTP layer that will parse exclusion wildcard expressions. The existing code issues 404s for wildcards as well as explicit indices. But, in general, in an expression with exclude wildcards (-...*) following other include wildcards, there is no way to tell if the include wildcard produced no results or they were subsequently excluded. Therefore, the proposed change is breaking the behavior of 404s for wildcards. Specifically, no 404s will be returned for wildcards, even if they are not followed by exclude wildcards or the exclude wildcards could not possibly exclude what has previously been included. Only explicitly requested aliases will be called out as missing. --- .../test/indices.get_alias/10_basic.yml | 4 +- .../test/indices.get_alias/30_wildcards.yml | 140 +++++++++++++++ .../admin/indices/RestGetAliasesAction.java | 164 ++++++++++-------- .../indices/RestGetAliasesActionTests.java | 134 ++++++++++++++ 4 files changed, 367 insertions(+), 75 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_alias/30_wildcards.yml create mode 100644 server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesActionTests.java diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_alias/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_alias/10_basic.yml index 6338598de05d0..f3dc3ac86bc9f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_alias/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_alias/10_basic.yml @@ -59,7 +59,7 @@ setup: - do: indices.get_alias: - name: _all + name: '*' - match: {test_index.aliases.test_alias: {}} - match: {test_index.aliases.test_blias: {}} @@ -220,7 +220,7 @@ setup: - is_false: test_index_2.aliases.test_blias --- -"Get aliases via /pref*/_alias/{name}": +"Get aliases via /*suf/_alias/{name}": - do: indices.get_alias: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_alias/30_wildcards.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_alias/30_wildcards.yml new file mode 100644 index 0000000000000..08b3009be0e88 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_alias/30_wildcards.yml @@ -0,0 +1,140 @@ +--- +setup: + + - do: + indices.create: + index: test_index + body: + aliases: + test_alias_1: {} + test_alias_2: {} + test_blias_1: {} + test_blias_2: {} + test: {} + +--- +"Get aliases wildcard and inclusion": + - do: + indices.get_alias: + name: test_alias*,test_blias_1 + + - match: {test_index.aliases.test_alias_1: {}} + - match: {test_index.aliases.test_alias_2: {}} + - match: {test_index.aliases.test_blias_1: {}} + - is_false: test_index.aliases.test_blias_2 + - is_false: test_index.aliases.test + +--- +"Get aliases wildcard and simple exclusion": + - skip: + version: " - 6.99.99" + reason: Exclusions in the alias expression are not handled + - do: + indices.get_alias: + name: test_blias_2,test_alias*,-test_alias_1 + + - is_false: test_index.aliases.test_alias_1 + - match: {test_index.aliases.test_alias_2: {}} + - is_false: test_index.aliases.test_blias_1 + - match: {test_index.aliases.test_blias_2: {}} + - is_false: test_index.aliases.test + +--- +"Get aliases and wildcard exclusion": + - skip: + version: " - 6.99.99" + reason: Exclusions in the alias expression are not handled + - do: + indices.get_alias: + name: test_alias_1,test_blias_1,-test_alias* + + - is_false: test_index.aliases.test_alias_1 + - is_false: test_index.aliases.test_alias_2 + - match: {test_index.aliases.test_blias_1: {}} + - is_false: test_index.aliases.test_blias_2 + - is_false: test_index.aliases.test + + - do: + indices.get_alias: + name: test_blias_2,tes*,-test_alias* + + - is_false: test_index.aliases.test_alias_1 + - is_false: test_index.aliases.test_alias_2 + - match: {test_index.aliases.test_blias_1: {}} + - match: {test_index.aliases.test_blias_2: {}} + - match: {test_index.aliases.test: {}} + +--- +"Non-existent exclusion alias before wildcard returns 404": + - skip: + version: " - 6.99.99" + reason: Exclusions in the alias expression are not handled + - do: + catch: missing + indices.get_alias: + name: -test_alias_1,test_alias*,-test_alias_2 + + - match: { 'status': 404} + - match: { 'error': 'alias [-test_alias_1] missing' } + - match: {test_index.aliases.test_alias_1: {}} + - is_false: test_index.aliases.test_alias_2 + - is_false: test_index.aliases.test_blias_1 + - is_false: test_index.aliases.test_blias_2 + - is_false: test_index.aliases.test + + - do: + catch: missing + indices.get_alias: + name: -test_alias_1,-non-existing,test_alias*,-test + + - match: { 'status': 404} + - match: { 'error': 'aliases [-non-existing,-test_alias_1] missing' } + - match: {test_index.aliases.test_alias_1: {}} + - match: {test_index.aliases.test_alias_2: {}} + - is_false: test_index.aliases.test_blias_1 + - is_false: test_index.aliases.test_blias_2 + - is_false: test_index.aliases.test + +--- +"Missing exclusions does not fire 404": + - skip: + version: " - 6.99.99" + reason: Exclusions in the alias expression are not handled + - do: + indices.get_alias: + name: test_alias*,-non-existent,test_blias*,-test + + - match: {test_index.aliases.test_alias_1: {}} + - match: {test_index.aliases.test_alias_2: {}} + - match: {test_index.aliases.test_blias_1: {}} + - match: {test_index.aliases.test_blias_2: {}} + - is_false: test_index.aliases.test + +--- +"Exclusion of non wildcarded aliases": + - skip: + version: " - 6.99.99" + reason: Exclusions in the alias expression are not handled + - do: + indices.get_alias: + name: test_alias_1,test_blias_2,-test_alias*,-test_blias_2 + + - match: { '': {}} + +--- +"Wildcard exclusions does not trigger 404": + - skip: + version: " - 6.99.99" + reason: Exclusions in the alias expression are not handled + - do: + catch: missing + indices.get_alias: + name: -non-existent,-non-existent*,-another + + - match: { 'status': 404} + - match: { 'error': 'alias [-non-existent] missing' } + - is_false: test_index.aliases.test_alias_1 + - is_false: test_index.aliases.test_alias_2 + - is_false: test_index.aliases.test_blias_1 + - is_false: test_index.aliases.test_blias_2 + - is_false: test_index.aliases.test diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java index 0d6d46e95b602..8cdf9e62b1096 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java @@ -25,11 +25,11 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.cluster.metadata.AliasMetaData; +import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.rest.BaseRestHandler; @@ -41,14 +41,12 @@ import org.elasticsearch.rest.action.RestBuilderListener; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Set; import java.util.SortedSet; -import java.util.stream.Collectors; +import java.util.TreeSet; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.HEAD; @@ -75,6 +73,94 @@ public String getName() { return "get_aliases_action"; } + static RestResponse buildRestResponse(boolean aliasesExplicitlyRequested, String[] requestedAliases, + ImmutableOpenMap> responseAliasMap, XContentBuilder builder) throws Exception { + final Set indicesToDisplay = new HashSet<>(); + final Set returnedAliasNames = new HashSet<>(); + for (final ObjectObjectCursor> cursor : responseAliasMap) { + for (final AliasMetaData aliasMetaData : cursor.value) { + if (aliasesExplicitlyRequested) { + // only display indices that have aliases + indicesToDisplay.add(cursor.key); + } + returnedAliasNames.add(aliasMetaData.alias()); + } + } + // compute explicitly requested aliases that have are not returned in the result + final SortedSet missingAliases = new TreeSet<>(); + // first wildcard index, leading "-" as an alias name after this index means + // that it is an exclusion + int firstWildcardIndex = requestedAliases.length; + for (int i = 0; i < requestedAliases.length; i++) { + if (Regex.isSimpleMatchPattern(requestedAliases[i])) { + firstWildcardIndex = i; + break; + } + } + for (int i = 0; i < requestedAliases.length; i++) { + if (MetaData.ALL.equals(requestedAliases[i]) || Regex.isSimpleMatchPattern(requestedAliases[i]) + || (i > firstWildcardIndex && requestedAliases[i].charAt(0) == '-')) { + // only explicitly requested aliases will be called out as missing (404) + continue; + } + // check if aliases[i] is subsequently excluded + int j = Math.max(i + 1, firstWildcardIndex); + for (; j < requestedAliases.length; j++) { + if (requestedAliases[j].charAt(0) == '-') { + // this is an exclude pattern + if (Regex.simpleMatch(requestedAliases[j].substring(1), requestedAliases[i]) + || MetaData.ALL.equals(requestedAliases[j].substring(1))) { + // aliases[i] is excluded by aliases[j] + break; + } + } + } + if (j == requestedAliases.length) { + // explicitly requested aliases[i] is not excluded by any subsequent "-" wildcard in expression + if (false == returnedAliasNames.contains(requestedAliases[i])) { + // aliases[i] is not in the result set + missingAliases.add(requestedAliases[i]); + } + } + } + + final RestStatus status; + builder.startObject(); + { + if (missingAliases.isEmpty()) { + status = RestStatus.OK; + } else { + status = RestStatus.NOT_FOUND; + final String message; + if (missingAliases.size() == 1) { + message = String.format(Locale.ROOT, "alias [%s] missing", Strings.collectionToCommaDelimitedString(missingAliases)); + } else { + message = String.format(Locale.ROOT, "aliases [%s] missing", Strings.collectionToCommaDelimitedString(missingAliases)); + } + builder.field("error", message); + builder.field("status", status.getStatus()); + } + + for (final ObjectObjectCursor> entry : responseAliasMap) { + if (aliasesExplicitlyRequested == false || (aliasesExplicitlyRequested && indicesToDisplay.contains(entry.key))) { + builder.startObject(entry.key); + { + builder.startObject("aliases"); + { + for (final AliasMetaData alias : entry.value) { + AliasMetaData.Builder.toXContent(alias, builder, ToXContent.EMPTY_PARAMS); + } + } + builder.endObject(); + } + builder.endObject(); + } + } + } + builder.endObject(); + return new BytesRestResponse(status, builder); + } + @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { // The TransportGetAliasesAction was improved do the same post processing as is happening here. @@ -94,76 +180,8 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC return channel -> client.admin().indices().getAliases(getAliasesRequest, new RestBuilderListener(channel) { @Override public RestResponse buildResponse(GetAliasesResponse response, XContentBuilder builder) throws Exception { - final ImmutableOpenMap> aliasMap = response.getAliases(); - - final Set aliasNames = new HashSet<>(); - final Set indicesToDisplay = new HashSet<>(); - for (final ObjectObjectCursor> cursor : aliasMap) { - for (final AliasMetaData aliasMetaData : cursor.value) { - aliasNames.add(aliasMetaData.alias()); - if (namesProvided) { - indicesToDisplay.add(cursor.key); - } - } - } - - // first remove requested aliases that are exact matches - final SortedSet difference = Sets.sortedDifference(Arrays.stream(aliases).collect(Collectors.toSet()), aliasNames); - - // now remove requested aliases that contain wildcards that are simple matches - final List matches = new ArrayList<>(); - outer: - for (final String pattern : difference) { - if (pattern.contains("*")) { - for (final String aliasName : aliasNames) { - if (Regex.simpleMatch(pattern, aliasName)) { - matches.add(pattern); - continue outer; - } - } - } - } - difference.removeAll(matches); - - final RestStatus status; - builder.startObject(); - { - if (difference.isEmpty()) { - status = RestStatus.OK; - } else { - status = RestStatus.NOT_FOUND; - final String message; - if (difference.size() == 1) { - message = String.format(Locale.ROOT, "alias [%s] missing", - Strings.collectionToCommaDelimitedString(difference)); - } else { - message = String.format(Locale.ROOT, "aliases [%s] missing", - Strings.collectionToCommaDelimitedString(difference)); - } - builder.field("error", message); - builder.field("status", status.getStatus()); - } - - for (final ObjectObjectCursor> entry : response.getAliases()) { - if (namesProvided == false || (namesProvided && indicesToDisplay.contains(entry.key))) { - builder.startObject(entry.key); - { - builder.startObject("aliases"); - { - for (final AliasMetaData alias : entry.value) { - AliasMetaData.Builder.toXContent(alias, builder, ToXContent.EMPTY_PARAMS); - } - } - builder.endObject(); - } - builder.endObject(); - } - } - } - builder.endObject(); - return new BytesRestResponse(status, builder); + return buildRestResponse(namesProvided, aliases, response.getAliases(), builder); } - }); } diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesActionTests.java new file mode 100644 index 0000000000000..ced52096687a2 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesActionTests.java @@ -0,0 +1,134 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.rest.action.admin.indices; + +import org.elasticsearch.cluster.metadata.AliasMetaData; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.rest.RestResponse; +import org.elasticsearch.test.ESTestCase; + +import java.util.Arrays; +import java.util.List; + +import static org.elasticsearch.rest.RestStatus.OK; +import static org.elasticsearch.rest.RestStatus.NOT_FOUND; +import static org.hamcrest.Matchers.equalTo; + +public class RestGetAliasesActionTests extends ESTestCase { + +// # Assumes the following setup +// curl -X PUT "localhost:9200/index" -H "Content-Type: application/json" -d' +// { +// "aliases": { +// "foo": {}, +// "foobar": {} +// } +// }' + + public void testBareRequest() throws Exception { + final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + final ImmutableOpenMap.Builder> openMapBuilder = ImmutableOpenMap.builder(); + final AliasMetaData foobarAliasMetaData = AliasMetaData.builder("foobar").build(); + final AliasMetaData fooAliasMetaData = AliasMetaData.builder("foo").build(); + openMapBuilder.put("index", Arrays.asList(fooAliasMetaData, foobarAliasMetaData)); + final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(false, new String[0], openMapBuilder.build(), + xContentBuilder); + assertThat(restResponse.status(), equalTo(OK)); + assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); + assertThat(restResponse.content().utf8ToString(), equalTo("{\"index\":{\"aliases\":{\"foo\":{},\"foobar\":{}}}}")); + } + + public void testSimpleAliasWildcardMatchingNothing() throws Exception { + final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + final ImmutableOpenMap.Builder> openMapBuilder = ImmutableOpenMap.builder(); + final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(true, new String[] { "baz*" }, openMapBuilder.build(), + xContentBuilder); + assertThat(restResponse.status(), equalTo(OK)); + assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); + assertThat(restResponse.content().utf8ToString(), equalTo("{}")); + } + + public void testMultipleAliasWildcardsSomeMatching() throws Exception { + final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + final ImmutableOpenMap.Builder> openMapBuilder = ImmutableOpenMap.builder(); + final AliasMetaData aliasMetaData = AliasMetaData.builder("foobar").build(); + openMapBuilder.put("index", Arrays.asList(aliasMetaData)); + final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(true, new String[] { "baz*", "foobar*" }, + openMapBuilder.build(), xContentBuilder); + assertThat(restResponse.status(), equalTo(OK)); + assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); + assertThat(restResponse.content().utf8ToString(), equalTo("{\"index\":{\"aliases\":{\"foobar\":{}}}}")); + } + + public void testAliasWildcardsIncludeAndExcludeAll() throws Exception { + final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + final ImmutableOpenMap.Builder> openMapBuilder = ImmutableOpenMap.builder(); + final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(true, new String[] { "foob*", "-foo*" }, + openMapBuilder.build(), xContentBuilder); + assertThat(restResponse.status(), equalTo(OK)); + assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); + assertThat(restResponse.content().utf8ToString(), equalTo("{}")); + } + + public void testAliasWildcardsIncludeAndExcludeSome() throws Exception { + final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + final ImmutableOpenMap.Builder> openMapBuilder = ImmutableOpenMap.builder(); + final AliasMetaData aliasMetaData = AliasMetaData.builder("foo").build(); + openMapBuilder.put("index", Arrays.asList(aliasMetaData)); + final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(true, new String[] { "foo*", "-foob*" }, + openMapBuilder.build(), xContentBuilder); + assertThat(restResponse.status(), equalTo(OK)); + assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); + assertThat(restResponse.content().utf8ToString(), equalTo("{\"index\":{\"aliases\":{\"foo\":{}}}}")); + } + + public void testAliasWildcardsIncludeAndExcludeSomeAndExplicitMissing() throws Exception { + final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + final ImmutableOpenMap.Builder> openMapBuilder = ImmutableOpenMap.builder(); + final AliasMetaData aliasMetaData = AliasMetaData.builder("foo").build(); + openMapBuilder.put("index", Arrays.asList(aliasMetaData)); + final String[] aliasPattern; + if (randomBoolean()) { + aliasPattern = new String[] { "missing", "foo*", "-foob*" }; + } else { + aliasPattern = new String[] { "foo*", "-foob*", "missing" }; + } + + final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(true, aliasPattern, openMapBuilder.build(), + xContentBuilder); + assertThat(restResponse.status(), equalTo(NOT_FOUND)); + assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); + assertThat(restResponse.content().utf8ToString(), + equalTo("{\"error\":\"alias [missing] missing\",\"status\":404,\"index\":{\"aliases\":{\"foo\":{}}}}")); + } + + public void testAliasWildcardsExcludeExplicitMissing() throws Exception { + final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + final ImmutableOpenMap.Builder> openMapBuilder = ImmutableOpenMap.builder(); + final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(true, new String[] { "foo", "foofoo", "-foo*" }, + openMapBuilder.build(), xContentBuilder); + assertThat(restResponse.status(), equalTo(OK)); + assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); + assertThat(restResponse.content().utf8ToString(), equalTo("{}")); + } +} From 00ace369af0c9b85dca78a57b083c65b7ffd823c Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 29 Jan 2019 11:47:29 -0700 Subject: [PATCH 021/100] Use `CcrRepository` to init follower index (#35719) This commit modifies the put follow index action to use a CcrRepository when creating a follower index. It routes the logic through the snapshot/restore process. A wait_for_active_shards parameter can be used to configure how long to wait before returning the response. --- .../client/CcrRequestConverters.java | 2 + .../client/ccr/PutFollowRequest.java | 18 +- .../java/org/elasticsearch/client/CCRIT.java | 3 +- .../documentation/CCRDocumentationIT.java | 14 +- .../high-level/ccr/put_follow.asciidoc | 2 + .../ccr/apis/follow/get-follow-info.asciidoc | 2 +- .../ccr/apis/follow/get-follow-stats.asciidoc | 2 +- .../apis/follow/post-pause-follow.asciidoc | 2 +- .../apis/follow/post-resume-follow.asciidoc | 2 +- .../ccr/apis/follow/post-unfollow.asciidoc | 2 +- .../ccr/apis/follow/put-follow.asciidoc | 9 +- .../reference/ccr/apis/get-ccr-stats.asciidoc | 2 +- docs/reference/ccr/getting-started.asciidoc | 2 +- .../restore/RestoreClusterStateListener.java | 87 ++++++++ .../TransportRestoreSnapshotAction.java | 43 +--- .../test/ccr/follow_and_unfollow.yml | 1 + .../rest-api-spec/test/ccr/follow_info.yml | 1 + .../rest-api-spec/test/ccr/follow_stats.yml | 1 + .../index_directly_into_follower_index.yml | 1 + .../xpack/ccr/ESCCRRestTestCase.java | 3 +- .../ccr/action/TransportPutFollowAction.java | 193 +++++++++--------- .../xpack/ccr/repository/CcrRepository.java | 18 +- .../xpack/ccr/rest/RestPutFollowAction.java | 4 +- .../elasticsearch/xpack/CcrIntegTestCase.java | 13 +- .../xpack/CcrSingleNodeTestCase.java | 2 + .../xpack/ccr/IndexFollowingIT.java | 147 ++++++++++++- .../xpack/ccr/LocalIndexFollowingIT.java | 6 +- .../action/PutFollowActionRequestTests.java | 3 +- .../core/ccr/action/PutFollowAction.java | 62 +++++- .../indexlifecycle/CCRIndexLifecycleIT.java | 5 +- .../rest-api-spec/api/ccr.follow.json | 7 + 31 files changed, 489 insertions(+), 170 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreClusterStateListener.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/CcrRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/CcrRequestConverters.java index df1e5dc01aef5..526db2a86a761 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/CcrRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/CcrRequestConverters.java @@ -46,6 +46,8 @@ static Request putFollow(PutFollowRequest putFollowRequest) throws IOException { .addPathPartAsIs("_ccr", "follow") .build(); Request request = new Request(HttpPut.METHOD_NAME, endpoint); + RequestConverters.Params parameters = new RequestConverters.Params(request); + parameters.withWaitForActiveShards(putFollowRequest.waitForActiveShards()); request.setEntity(createEntity(putFollowRequest, REQUEST_BODY_CONTENT_TYPE)); return request; } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/PutFollowRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/PutFollowRequest.java index 98e9d224564cf..8307b04bd7087 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/PutFollowRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/PutFollowRequest.java @@ -19,6 +19,7 @@ package org.elasticsearch.client.ccr; +import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.Validatable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.xcontent.ToXContentObject; @@ -36,11 +37,17 @@ public final class PutFollowRequest extends FollowConfig implements Validatable, private final String remoteCluster; private final String leaderIndex; private final String followerIndex; + private final ActiveShardCount waitForActiveShards; public PutFollowRequest(String remoteCluster, String leaderIndex, String followerIndex) { + this(remoteCluster, leaderIndex, followerIndex, ActiveShardCount.NONE); + } + + public PutFollowRequest(String remoteCluster, String leaderIndex, String followerIndex, ActiveShardCount waitForActiveShards) { this.remoteCluster = Objects.requireNonNull(remoteCluster, "remoteCluster"); this.leaderIndex = Objects.requireNonNull(leaderIndex, "leaderIndex"); this.followerIndex = Objects.requireNonNull(followerIndex, "followerIndex"); + this.waitForActiveShards = waitForActiveShards; } @Override @@ -66,13 +73,18 @@ public String getFollowerIndex() { return followerIndex; } + public ActiveShardCount waitForActiveShards() { + return waitForActiveShards; + } + @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; if (!super.equals(o)) return false; PutFollowRequest that = (PutFollowRequest) o; - return Objects.equals(remoteCluster, that.remoteCluster) && + return Objects.equals(waitForActiveShards, that.waitForActiveShards) && + Objects.equals(remoteCluster, that.remoteCluster) && Objects.equals(leaderIndex, that.leaderIndex) && Objects.equals(followerIndex, that.followerIndex); } @@ -83,7 +95,7 @@ public int hashCode() { super.hashCode(), remoteCluster, leaderIndex, - followerIndex - ); + followerIndex, + waitForActiveShards); } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/CCRIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/CCRIT.java index 97a379aa16a90..ee2685dee6d92 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/CCRIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/CCRIT.java @@ -27,6 +27,7 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.ccr.CcrStatsRequest; import org.elasticsearch.client.ccr.CcrStatsResponse; @@ -95,7 +96,7 @@ public void testIndexFollowing() throws Exception { CreateIndexResponse response = highLevelClient().indices().create(createIndexRequest, RequestOptions.DEFAULT); assertThat(response.isAcknowledged(), is(true)); - PutFollowRequest putFollowRequest = new PutFollowRequest("local_cluster", "leader", "follower"); + PutFollowRequest putFollowRequest = new PutFollowRequest("local_cluster", "leader", "follower", ActiveShardCount.ONE); PutFollowResponse putFollowResponse = execute(putFollowRequest, ccrClient::putFollow, ccrClient::putFollowAsync); assertThat(putFollowResponse.isFollowIndexCreated(), is(true)); assertThat(putFollowResponse.isFollowIndexShardsAcked(), is(true)); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CCRDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CCRDocumentationIT.java index 9ef01a1a6f7bf..2e54d1c4a1a7c 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CCRDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CCRDocumentationIT.java @@ -26,6 +26,7 @@ import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.ESRestHighLevelClientTestCase; import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; @@ -97,7 +98,8 @@ public void testPutFollow() throws Exception { PutFollowRequest putFollowRequest = new PutFollowRequest( "local", // <1> "leader", // <2> - "follower" // <3> + "follower", // <3> + ActiveShardCount.ONE // <4> ); // end::ccr-put-follow-request @@ -175,7 +177,7 @@ public void testPauseFollow() throws Exception { String followIndex = "follower"; // Follow index, so that it can be paused: { - PutFollowRequest putFollowRequest = new PutFollowRequest("local", "leader", followIndex); + PutFollowRequest putFollowRequest = new PutFollowRequest("local", "leader", followIndex, ActiveShardCount.ONE); PutFollowResponse putFollowResponse = client.ccr().putFollow(putFollowRequest, RequestOptions.DEFAULT); assertThat(putFollowResponse.isFollowIndexCreated(), is(true)); assertThat(putFollowResponse.isFollowIndexShardsAcked(), is(true)); @@ -241,7 +243,7 @@ public void testResumeFollow() throws Exception { String followIndex = "follower"; // Follow index, so that it can be paused: { - PutFollowRequest putFollowRequest = new PutFollowRequest("local", "leader", followIndex); + PutFollowRequest putFollowRequest = new PutFollowRequest("local", "leader", followIndex, ActiveShardCount.ONE); PutFollowResponse putFollowResponse = client.ccr().putFollow(putFollowRequest, RequestOptions.DEFAULT); assertThat(putFollowResponse.isFollowIndexCreated(), is(true)); assertThat(putFollowResponse.isFollowIndexShardsAcked(), is(true)); @@ -317,7 +319,7 @@ public void testUnfollow() throws Exception { String followIndex = "follower"; // Follow index, pause and close, so that it can be unfollowed: { - PutFollowRequest putFollowRequest = new PutFollowRequest("local", "leader", followIndex); + PutFollowRequest putFollowRequest = new PutFollowRequest("local", "leader", followIndex, ActiveShardCount.ONE); PutFollowResponse putFollowResponse = client.ccr().putFollow(putFollowRequest, RequestOptions.DEFAULT); assertThat(putFollowResponse.isFollowIndexCreated(), is(true)); assertThat(putFollowResponse.isFollowIndexShardsAcked(), is(true)); @@ -349,7 +351,7 @@ public void testUnfollow() throws Exception { DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(followIndex); assertThat(client.indices().delete(deleteIndexRequest, RequestOptions.DEFAULT).isAcknowledged(), is(true)); - PutFollowRequest putFollowRequest = new PutFollowRequest("local", "leader", followIndex); + PutFollowRequest putFollowRequest = new PutFollowRequest("local", "leader", followIndex, ActiveShardCount.ONE); PutFollowResponse putFollowResponse = client.ccr().putFollow(putFollowRequest, RequestOptions.DEFAULT); assertThat(putFollowResponse.isFollowIndexCreated(), is(true)); assertThat(putFollowResponse.isFollowIndexShardsAcked(), is(true)); @@ -639,7 +641,7 @@ public void testGetFollowStats() throws Exception { } { // Follow index, so that we can query for follow stats: - PutFollowRequest putFollowRequest = new PutFollowRequest("local", "leader", "follower"); + PutFollowRequest putFollowRequest = new PutFollowRequest("local", "leader", "follower", ActiveShardCount.ONE); PutFollowResponse putFollowResponse = client.ccr().putFollow(putFollowRequest, RequestOptions.DEFAULT); assertThat(putFollowResponse.isFollowIndexCreated(), is(true)); assertThat(putFollowResponse.isFollowIndexShardsAcked(), is(true)); diff --git a/docs/java-rest/high-level/ccr/put_follow.asciidoc b/docs/java-rest/high-level/ccr/put_follow.asciidoc index 839a9bedc6552..2f40bbd5d2b2d 100644 --- a/docs/java-rest/high-level/ccr/put_follow.asciidoc +++ b/docs/java-rest/high-level/ccr/put_follow.asciidoc @@ -20,6 +20,8 @@ include-tagged::{doc-tests-file}[{api}-request] <1> The name of the remote cluster alias. <2> The name of the leader in the remote cluster. <3> The name of the follower index that gets created as part of the put follow API call. +<4> The number of active shard copies to wait for before the put follow API returns a +response, as an `ActiveShardCount` [id="{upid}-{api}-response"] ==== Response diff --git a/docs/reference/ccr/apis/follow/get-follow-info.asciidoc b/docs/reference/ccr/apis/follow/get-follow-info.asciidoc index 22418db10887c..212b1167b6e33 100644 --- a/docs/reference/ccr/apis/follow/get-follow-info.asciidoc +++ b/docs/reference/ccr/apis/follow/get-follow-info.asciidoc @@ -22,7 +22,7 @@ replication options and whether the follower indices are active or paused. [source,js] -------------------------------------------------- -PUT /follower_index/_ccr/follow +PUT /follower_index/_ccr/follow?wait_for_active_shards=1 { "remote_cluster" : "remote_cluster", "leader_index" : "leader_index" diff --git a/docs/reference/ccr/apis/follow/get-follow-stats.asciidoc b/docs/reference/ccr/apis/follow/get-follow-stats.asciidoc index 766f502ff93a3..8c02582e01278 100644 --- a/docs/reference/ccr/apis/follow/get-follow-stats.asciidoc +++ b/docs/reference/ccr/apis/follow/get-follow-stats.asciidoc @@ -21,7 +21,7 @@ following tasks associated with each shard for the specified indices. [source,js] -------------------------------------------------- -PUT /follower_index/_ccr/follow +PUT /follower_index/_ccr/follow?wait_for_active_shards=1 { "remote_cluster" : "remote_cluster", "leader_index" : "leader_index" diff --git a/docs/reference/ccr/apis/follow/post-pause-follow.asciidoc b/docs/reference/ccr/apis/follow/post-pause-follow.asciidoc index 0d56ee76bd9b9..f5b0bef7b2994 100644 --- a/docs/reference/ccr/apis/follow/post-pause-follow.asciidoc +++ b/docs/reference/ccr/apis/follow/post-pause-follow.asciidoc @@ -24,7 +24,7 @@ following task. [source,js] -------------------------------------------------- -PUT /follower_index/_ccr/follow +PUT /follower_index/_ccr/follow?wait_for_active_shards=1 { "remote_cluster" : "remote_cluster", "leader_index" : "leader_index" diff --git a/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc b/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc index e8b4cd50f27e7..736061f2bfde8 100644 --- a/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc +++ b/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc @@ -23,7 +23,7 @@ returns, the follower index will resume fetching operations from the leader inde [source,js] -------------------------------------------------- -PUT /follower_index/_ccr/follow +PUT /follower_index/_ccr/follow?wait_for_active_shards=1 { "remote_cluster" : "remote_cluster", "leader_index" : "leader_index" diff --git a/docs/reference/ccr/apis/follow/post-unfollow.asciidoc b/docs/reference/ccr/apis/follow/post-unfollow.asciidoc index 6507c04ac5026..c3126d02d1efc 100644 --- a/docs/reference/ccr/apis/follow/post-unfollow.asciidoc +++ b/docs/reference/ccr/apis/follow/post-unfollow.asciidoc @@ -27,7 +27,7 @@ irreversible operation. [source,js] -------------------------------------------------- -PUT /follower_index/_ccr/follow +PUT /follower_index/_ccr/follow?wait_for_active_shards=1 { "remote_cluster" : "remote_cluster", "leader_index" : "leader_index" diff --git a/docs/reference/ccr/apis/follow/put-follow.asciidoc b/docs/reference/ccr/apis/follow/put-follow.asciidoc index 3f6156c1e6820..52253d6ad2f4c 100644 --- a/docs/reference/ccr/apis/follow/put-follow.asciidoc +++ b/docs/reference/ccr/apis/follow/put-follow.asciidoc @@ -31,7 +31,7 @@ POST /follower_index/_ccr/pause_follow [source,js] -------------------------------------------------- -PUT //_ccr/follow +PUT //_ccr/follow?wait_for_active_shards=1 { "remote_cluster" : "", "leader_index" : "" @@ -43,6 +43,11 @@ PUT //_ccr/follow // TEST[s//remote_cluster/] // TEST[s//leader_index/] +The `wait_for_active_shards` parameter specifies the number of shards to wait on being active +before responding. This defaults to waiting on none of the shards to be active. A shard must +be restored from the leader index being active. Restoring a follower shard requires transferring +all the remote Lucene segment files to the follower index. + ==== Path Parameters `follower_index` (required):: @@ -73,7 +78,7 @@ This example creates a follower index named `follower_index`: [source,js] -------------------------------------------------- -PUT /follower_index/_ccr/follow +PUT /follower_index/_ccr/follow?wait_for_active_shards=1 { "remote_cluster" : "remote_cluster", "leader_index" : "leader_index", diff --git a/docs/reference/ccr/apis/get-ccr-stats.asciidoc b/docs/reference/ccr/apis/get-ccr-stats.asciidoc index f47e49ee82674..8949de8787fa7 100644 --- a/docs/reference/ccr/apis/get-ccr-stats.asciidoc +++ b/docs/reference/ccr/apis/get-ccr-stats.asciidoc @@ -22,7 +22,7 @@ shard-level stats as in the <>. [source,js] -------------------------------------------------- -PUT /follower_index/_ccr/follow +PUT /follower_index/_ccr/follow?wait_for_active_shards=1 { "remote_cluster" : "remote_cluster", "leader_index" : "leader_index" diff --git a/docs/reference/ccr/getting-started.asciidoc b/docs/reference/ccr/getting-started.asciidoc index 1af236f7d86fb..7c59b8628052f 100644 --- a/docs/reference/ccr/getting-started.asciidoc +++ b/docs/reference/ccr/getting-started.asciidoc @@ -230,7 +230,7 @@ cluster. [source,js] -------------------------------------------------- -PUT /server-metrics-copy/_ccr/follow +PUT /server-metrics-copy/_ccr/follow?wait_for_active_shards=1 { "remote_cluster" : "leader", "leader_index" : "server-metrics" diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreClusterStateListener.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreClusterStateListener.java new file mode 100644 index 0000000000000..a74aad3ddb586 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreClusterStateListener.java @@ -0,0 +1,87 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.cluster.snapshots.restore; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.RestoreInProgress; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.snapshots.RestoreInfo; +import org.elasticsearch.snapshots.RestoreService; + +import static org.elasticsearch.snapshots.RestoreService.restoreInProgress; + +public class RestoreClusterStateListener implements ClusterStateListener { + + private static final Logger logger = LogManager.getLogger(RestoreClusterStateListener.class); + + private final ClusterService clusterService; + private final String uuid; + private final ActionListener listener; + + + private RestoreClusterStateListener(ClusterService clusterService, RestoreService.RestoreCompletionResponse response, + ActionListener listener) { + this.clusterService = clusterService; + this.uuid = response.getUuid(); + this.listener = listener; + } + + @Override + public void clusterChanged(ClusterChangedEvent changedEvent) { + final RestoreInProgress.Entry prevEntry = restoreInProgress(changedEvent.previousState(), uuid); + final RestoreInProgress.Entry newEntry = restoreInProgress(changedEvent.state(), uuid); + if (prevEntry == null) { + // When there is a master failure after a restore has been started, this listener might not be registered + // on the current master and as such it might miss some intermediary cluster states due to batching. + // Clean up listener in that case and acknowledge completion of restore operation to client. + clusterService.removeListener(this); + listener.onResponse(new RestoreSnapshotResponse(null)); + } else if (newEntry == null) { + clusterService.removeListener(this); + ImmutableOpenMap shards = prevEntry.shards(); + assert prevEntry.state().completed() : "expected completed snapshot state but was " + prevEntry.state(); + assert RestoreService.completed(shards) : "expected all restore entries to be completed"; + RestoreInfo ri = new RestoreInfo(prevEntry.snapshot().getSnapshotId().getName(), + prevEntry.indices(), + shards.size(), + shards.size() - RestoreService.failedShards(shards)); + RestoreSnapshotResponse response = new RestoreSnapshotResponse(ri); + logger.debug("restore of [{}] completed", prevEntry.snapshot().getSnapshotId()); + listener.onResponse(response); + } else { + // restore not completed yet, wait for next cluster state update + } + } + + /** + * Creates a cluster state listener and registers it with the cluster service. The listener passed as a + * parameter will be called when the restore is complete. + */ + public static void createAndRegisterListener(ClusterService clusterService, RestoreService.RestoreCompletionResponse response, + ActionListener listener) { + clusterService.addListener(new RestoreClusterStateListener(clusterService, response, listener)); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java index b362be49b10ab..d8dcc5eb8f846 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java @@ -22,26 +22,17 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.master.TransportMasterNodeAction; -import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.ClusterStateListener; -import org.elasticsearch.cluster.RestoreInProgress; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.snapshots.RestoreInfo; import org.elasticsearch.snapshots.RestoreService; import org.elasticsearch.snapshots.RestoreService.RestoreCompletionResponse; -import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import static org.elasticsearch.snapshots.RestoreService.restoreInProgress; - /** * Transport action for restore snapshot operation */ @@ -86,39 +77,7 @@ protected void masterOperation(final RestoreSnapshotRequest request, final Clust @Override public void onResponse(RestoreCompletionResponse restoreCompletionResponse) { if (restoreCompletionResponse.getRestoreInfo() == null && request.waitForCompletion()) { - final Snapshot snapshot = restoreCompletionResponse.getSnapshot(); - String uuid = restoreCompletionResponse.getUuid(); - - ClusterStateListener clusterStateListener = new ClusterStateListener() { - @Override - public void clusterChanged(ClusterChangedEvent changedEvent) { - final RestoreInProgress.Entry prevEntry = restoreInProgress(changedEvent.previousState(), uuid); - final RestoreInProgress.Entry newEntry = restoreInProgress(changedEvent.state(), uuid); - if (prevEntry == null) { - // When there is a master failure after a restore has been started, this listener might not be registered - // on the current master and as such it might miss some intermediary cluster states due to batching. - // Clean up listener in that case and acknowledge completion of restore operation to client. - clusterService.removeListener(this); - listener.onResponse(new RestoreSnapshotResponse(null)); - } else if (newEntry == null) { - clusterService.removeListener(this); - ImmutableOpenMap shards = prevEntry.shards(); - assert prevEntry.state().completed() : "expected completed snapshot state but was " + prevEntry.state(); - assert RestoreService.completed(shards) : "expected all restore entries to be completed"; - RestoreInfo ri = new RestoreInfo(prevEntry.snapshot().getSnapshotId().getName(), - prevEntry.indices(), - shards.size(), - shards.size() - RestoreService.failedShards(shards)); - RestoreSnapshotResponse response = new RestoreSnapshotResponse(ri); - logger.debug("restore of [{}] completed", snapshot); - listener.onResponse(response); - } else { - // restore not completed yet, wait for next cluster state update - } - } - }; - - clusterService.addListener(clusterStateListener); + RestoreClusterStateListener.createAndRegisterListener(clusterService, restoreCompletionResponse, listener); } else { listener.onResponse(new RestoreSnapshotResponse(restoreCompletionResponse.getRestoreInfo())); } diff --git a/x-pack/plugin/ccr/qa/rest/src/test/resources/rest-api-spec/test/ccr/follow_and_unfollow.yml b/x-pack/plugin/ccr/qa/rest/src/test/resources/rest-api-spec/test/ccr/follow_and_unfollow.yml index f73f5c6dfb2d3..d5cd8ebd4f1ab 100644 --- a/x-pack/plugin/ccr/qa/rest/src/test/resources/rest-api-spec/test/ccr/follow_and_unfollow.yml +++ b/x-pack/plugin/ccr/qa/rest/src/test/resources/rest-api-spec/test/ccr/follow_and_unfollow.yml @@ -37,6 +37,7 @@ - do: ccr.follow: index: bar + wait_for_active_shards: 1 body: remote_cluster: local leader_index: foo diff --git a/x-pack/plugin/ccr/qa/rest/src/test/resources/rest-api-spec/test/ccr/follow_info.yml b/x-pack/plugin/ccr/qa/rest/src/test/resources/rest-api-spec/test/ccr/follow_info.yml index f1e47d830cf97..8383ecd4e6851 100644 --- a/x-pack/plugin/ccr/qa/rest/src/test/resources/rest-api-spec/test/ccr/follow_info.yml +++ b/x-pack/plugin/ccr/qa/rest/src/test/resources/rest-api-spec/test/ccr/follow_info.yml @@ -33,6 +33,7 @@ - do: ccr.follow: index: bar + wait_for_active_shards: 1 body: remote_cluster: local leader_index: foo diff --git a/x-pack/plugin/ccr/qa/rest/src/test/resources/rest-api-spec/test/ccr/follow_stats.yml b/x-pack/plugin/ccr/qa/rest/src/test/resources/rest-api-spec/test/ccr/follow_stats.yml index 5b3e6c18ef29b..220463a60b258 100644 --- a/x-pack/plugin/ccr/qa/rest/src/test/resources/rest-api-spec/test/ccr/follow_stats.yml +++ b/x-pack/plugin/ccr/qa/rest/src/test/resources/rest-api-spec/test/ccr/follow_stats.yml @@ -36,6 +36,7 @@ - do: ccr.follow: index: bar + wait_for_active_shards: 1 body: remote_cluster: local leader_index: foo diff --git a/x-pack/plugin/ccr/qa/rest/src/test/resources/rest-api-spec/test/ccr/index_directly_into_follower_index.yml b/x-pack/plugin/ccr/qa/rest/src/test/resources/rest-api-spec/test/ccr/index_directly_into_follower_index.yml index 60c3b404b6f09..62878437c37e3 100644 --- a/x-pack/plugin/ccr/qa/rest/src/test/resources/rest-api-spec/test/ccr/index_directly_into_follower_index.yml +++ b/x-pack/plugin/ccr/qa/rest/src/test/resources/rest-api-spec/test/ccr/index_directly_into_follower_index.yml @@ -37,6 +37,7 @@ - do: ccr.follow: index: bar + wait_for_active_shards: 1 body: remote_cluster: local leader_index: foo diff --git a/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java b/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java index 25fbef7ada73e..6cdb6b37961ff 100644 --- a/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java +++ b/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java @@ -73,7 +73,7 @@ protected static void followIndex(String leaderCluster, String leaderIndex, Stri } protected static void followIndex(RestClient client, String leaderCluster, String leaderIndex, String followIndex) throws IOException { - final Request request = new Request("PUT", "/" + followIndex + "/_ccr/follow"); + final Request request = new Request("PUT", "/" + followIndex + "/_ccr/follow?wait_for_active_shards=1"); request.setJsonEntity("{\"remote_cluster\": \"" + leaderCluster + "\", \"leader_index\": \"" + leaderIndex + "\", \"read_poll_timeout\": \"10ms\"}"); assertOK(client.performRequest(request)); @@ -186,6 +186,7 @@ protected static Map toMap(String response) { protected static void ensureYellow(String index) throws IOException { Request request = new Request("GET", "/_cluster/health/" + index); request.addParameter("wait_for_status", "yellow"); + request.addParameter("wait_for_active_shards", "1"); request.addParameter("wait_for_no_relocating_shards", "true"); request.addParameter("wait_for_no_initializing_shards", "true"); request.addParameter("timeout", "70s"); diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java index bca8608904a76..27f3b60fb5291 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java @@ -6,52 +6,53 @@ package org.elasticsearch.xpack.ccr.action; -import com.carrotsearch.hppc.cursors.ObjectObjectCursor; -import org.elasticsearch.ResourceAlreadyExistsException; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreClusterStateListener; +import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; +import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.ActiveShardsObserver; import org.elasticsearch.action.support.master.TransportMasterNodeAction; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.AckedClusterStateUpdateTask; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.cluster.routing.RoutingTable; -import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.license.LicenseUtils; +import org.elasticsearch.snapshots.RestoreInfo; +import org.elasticsearch.snapshots.RestoreService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xpack.ccr.Ccr; import org.elasticsearch.xpack.ccr.CcrLicenseChecker; import org.elasticsearch.xpack.ccr.CcrSettings; +import org.elasticsearch.xpack.ccr.repository.CcrRepository; import org.elasticsearch.xpack.core.ccr.action.PutFollowAction; import org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction; import java.io.IOException; -import java.util.HashMap; -import java.util.Map; import java.util.Objects; public final class TransportPutFollowAction - extends TransportMasterNodeAction { + extends TransportMasterNodeAction { + + private static final Logger logger = LogManager.getLogger(TransportPutFollowAction.class); private final Client client; - private final AllocationService allocationService; - private final ActiveShardsObserver activeShardsObserver; + private final RestoreService restoreService; private final CcrLicenseChecker ccrLicenseChecker; + private final ActiveShardsObserver activeShardsObserver; @Inject public TransportPutFollowAction( @@ -61,7 +62,7 @@ public TransportPutFollowAction( final ActionFilters actionFilters, final IndexNameExpressionResolver indexNameExpressionResolver, final Client client, - final AllocationService allocationService, + final RestoreService restoreService, final CcrLicenseChecker ccrLicenseChecker) { super( PutFollowAction.NAME, @@ -72,9 +73,9 @@ public TransportPutFollowAction( PutFollowAction.Request::new, indexNameExpressionResolver); this.client = client; - this.allocationService = allocationService; - this.activeShardsObserver = new ActiveShardsObserver(clusterService, threadPool); + this.restoreService = restoreService; this.ccrLicenseChecker = Objects.requireNonNull(ccrLicenseChecker); + this.activeShardsObserver = new ActiveShardsObserver(clusterService, threadPool); } @Override @@ -96,7 +97,7 @@ protected PutFollowAction.Response read(StreamInput in) throws IOException { protected void masterOperation( final PutFollowAction.Request request, final ClusterState state, - final ActionListener listener) throws Exception { + final ActionListener listener) { if (ccrLicenseChecker.isCcrAllowed() == false) { listener.onFailure(LicenseUtils.newComplianceException("ccr")); return; @@ -111,12 +112,11 @@ protected void masterOperation( remoteCluster, leaderIndex, listener::onFailure, - (historyUUID, leaderIndexMetaData) -> createFollowerIndex(leaderIndexMetaData, historyUUID, request, listener)); + (historyUUID, leaderIndexMetaData) -> createFollowerIndex(leaderIndexMetaData, request, listener)); } private void createFollowerIndex( final IndexMetaData leaderIndexMetaData, - final String[] historyUUIDs, final PutFollowAction.Request request, final ActionListener listener) { if (leaderIndexMetaData == null) { @@ -131,98 +131,107 @@ private void createFollowerIndex( return; } - ActionListener handler = ActionListener.wrap( - result -> { - if (result) { - initiateFollowing(request, listener); - } else { - listener.onResponse(new PutFollowAction.Response(true, false, false)); - } - }, - listener::onFailure); - // Can't use create index api here, because then index templates can alter the mappings / settings. - // And index templates could introduce settings / mappings that are incompatible with the leader index. - clusterService.submitStateUpdateTask("create_following_index", new AckedClusterStateUpdateTask(request, handler) { + final Settings.Builder settingsBuilder = Settings.builder() + .put(IndexMetaData.SETTING_INDEX_PROVIDED_NAME, request.getFollowRequest().getFollowerIndex()) + .put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true); + final String leaderClusterRepoName = CcrRepository.NAME_PREFIX + request.getRemoteCluster(); + final RestoreSnapshotRequest restoreRequest = new RestoreSnapshotRequest(leaderClusterRepoName, CcrRepository.LATEST) + .indices(request.getLeaderIndex()).indicesOptions(request.indicesOptions()).renamePattern("^(.*)$") + .renameReplacement(request.getFollowRequest().getFollowerIndex()).masterNodeTimeout(request.masterNodeTimeout()) + .indexSettings(settingsBuilder); + + final Client clientWithHeaders = CcrLicenseChecker.wrapClient(this.client, threadPool.getThreadContext().getHeaders()); + threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(new AbstractRunnable() { @Override - protected Boolean newResponse(final boolean acknowledged) { - return acknowledged; + public void onFailure(Exception e) { + listener.onFailure(e); } @Override - public ClusterState execute(final ClusterState currentState) throws Exception { - String followIndex = request.getFollowRequest().getFollowerIndex(); - IndexMetaData currentIndex = currentState.metaData().index(followIndex); - if (currentIndex != null) { - throw new ResourceAlreadyExistsException(currentIndex.getIndex()); - } + protected void doRun() throws Exception { + restoreService.restoreSnapshot(restoreRequest, new ActionListener() { - MetaData.Builder mdBuilder = MetaData.builder(currentState.metaData()); - IndexMetaData.Builder imdBuilder = IndexMetaData.builder(followIndex); - - // Adding the leader index uuid for each shard as custom metadata: - Map metadata = new HashMap<>(); - metadata.put(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_SHARD_HISTORY_UUIDS, String.join(",", historyUUIDs)); - metadata.put(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY, leaderIndexMetaData.getIndexUUID()); - metadata.put(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_NAME_KEY, leaderIndexMetaData.getIndex().getName()); - metadata.put(Ccr.CCR_CUSTOM_METADATA_REMOTE_CLUSTER_NAME_KEY, request.getRemoteCluster()); - imdBuilder.putCustom(Ccr.CCR_CUSTOM_METADATA_KEY, metadata); - - // Copy all settings, but overwrite a few settings. - Settings.Builder settingsBuilder = Settings.builder(); - settingsBuilder.put(leaderIndexMetaData.getSettings()); - // Overwriting UUID here, because otherwise we can't follow indices in the same cluster - settingsBuilder.put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()); - settingsBuilder.put(IndexMetaData.SETTING_INDEX_PROVIDED_NAME, followIndex); - settingsBuilder.put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true); - settingsBuilder.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true); - imdBuilder.settings(settingsBuilder); - - // Copy mappings from leader IMD to follow IMD - for (ObjectObjectCursor cursor : leaderIndexMetaData.getMappings()) { - imdBuilder.putMapping(cursor.value); - } - imdBuilder.setRoutingNumShards(leaderIndexMetaData.getRoutingNumShards()); - IndexMetaData followIMD = imdBuilder.build(); - mdBuilder.put(followIMD, false); + @Override + public void onResponse(RestoreService.RestoreCompletionResponse response) { + afterRestoreStarted(clientWithHeaders, request, listener, response); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } + }); + } - ClusterState.Builder builder = ClusterState.builder(currentState); - builder.metaData(mdBuilder.build()); - ClusterState updatedState = builder.build(); + private void afterRestoreStarted(Client clientWithHeaders, PutFollowAction.Request request, + ActionListener originalListener, + RestoreService.RestoreCompletionResponse response) { + final ActionListener listener; + if (ActiveShardCount.NONE.equals(request.waitForActiveShards())) { + originalListener.onResponse(new PutFollowAction.Response(true, false, false)); + listener = new ActionListener() { + + @Override + public void onResponse(PutFollowAction.Response response) { + logger.debug("put follow {} completed with {}", request, response); + } - RoutingTable.Builder routingTableBuilder = RoutingTable.builder(updatedState.routingTable()) - .addAsNew(updatedState.metaData().index(request.getFollowRequest().getFollowerIndex())); - updatedState = allocationService.reroute( - ClusterState.builder(updatedState).routingTable(routingTableBuilder.build()).build(), - "follow index [" + request.getFollowRequest().getFollowerIndex() + "] created"); + @Override + public void onFailure(Exception e) { + logger.debug(() -> new ParameterizedMessage("put follow {} failed during the restore process", request), e); + } + }; + } else { + listener = originalListener; + } - logger.info("[{}] creating index, cause [ccr_create_and_follow], shards [{}]/[{}]", - followIndex, followIMD.getNumberOfShards(), followIMD.getNumberOfReplicas()); + RestoreClusterStateListener.createAndRegisterListener(clusterService, response, new ActionListener() { + @Override + public void onResponse(RestoreSnapshotResponse restoreSnapshotResponse) { + RestoreInfo restoreInfo = restoreSnapshotResponse.getRestoreInfo(); + + if (restoreInfo == null) { + // If restoreInfo is null then it is possible there was a master failure during the + // restore. + listener.onResponse(new PutFollowAction.Response(true, false, false)); + } else if (restoreInfo.failedShards() == 0) { + initiateFollowing(clientWithHeaders, request, listener); + } else { + assert restoreInfo.failedShards() > 0 : "Should have failed shards"; + listener.onResponse(new PutFollowAction.Response(true, false, false)); + } + } - return updatedState; + @Override + public void onFailure(Exception e) { + listener.onFailure(e); } }); } private void initiateFollowing( - final PutFollowAction.Request request, - final ActionListener listener) { + final Client client, + final PutFollowAction.Request request, + final ActionListener listener) { + assert request.waitForActiveShards() != ActiveShardCount.DEFAULT : "PutFollowAction does not support DEFAULT."; activeShardsObserver.waitForActiveShards(new String[]{request.getFollowRequest().getFollowerIndex()}, - ActiveShardCount.DEFAULT, request.timeout(), result -> { - if (result) { - client.execute(ResumeFollowAction.INSTANCE, request.getFollowRequest(), ActionListener.wrap( - r -> listener.onResponse(new PutFollowAction.Response(true, true, r.isAcknowledged())), - listener::onFailure - )); - } else { - listener.onResponse(new PutFollowAction.Response(true, false, false)); - } - }, listener::onFailure); + request.waitForActiveShards(), request.timeout(), result -> { + if (result) { + client.execute(ResumeFollowAction.INSTANCE, request.getFollowRequest(), ActionListener.wrap( + r -> listener.onResponse(new PutFollowAction.Response(true, true, r.isAcknowledged())), + listener::onFailure + )); + } else { + listener.onResponse(new PutFollowAction.Response(true, false, false)); + } + }, listener::onFailure); } @Override protected ClusterBlockException checkBlock(final PutFollowAction.Request request, final ClusterState state) { return state.blocks().indexBlockedException(ClusterBlockLevel.METADATA_WRITE, request.getFollowRequest().getFollowerIndex()); } - } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java index 33a8c64c96138..7ca95a14909ae 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java @@ -6,6 +6,8 @@ package org.elasticsearch.xpack.ccr.repository; +import com.carrotsearch.hppc.cursors.IntObjectCursor; +import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.apache.lucene.index.IndexCommit; import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; @@ -81,6 +83,7 @@ public class CcrRepository extends AbstractLifecycleComponent implements Reposit public static final String TYPE = "_ccr_"; public static final String NAME_PREFIX = "_ccr_"; private static final SnapshotId SNAPSHOT_ID = new SnapshotId(LATEST, LATEST); + private static final String IN_SYNC_ALLOCATION_ID = "ccr_restore"; private final RepositoryMetaData metadata; private final CcrSettings ccrSettings; @@ -157,7 +160,7 @@ public IndexMetaData getSnapshotIndexMetaData(SnapshotId snapshotId, IndexId ind ccrLicenseChecker.fetchLeaderHistoryUUIDs(remoteClient, leaderIndexMetaData, future::onFailure, future::onResponse); String[] leaderHistoryUUIDs = future.actionGet(); - IndexMetaData.Builder imdBuilder = IndexMetaData.builder(leaderIndexMetaData); + IndexMetaData.Builder imdBuilder = IndexMetaData.builder(leaderIndex); // Adding the leader index uuid for each shard as custom metadata: Map metadata = new HashMap<>(); metadata.put(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_SHARD_HISTORY_UUIDS, String.join(",", leaderHistoryUUIDs)); @@ -166,6 +169,19 @@ public IndexMetaData getSnapshotIndexMetaData(SnapshotId snapshotId, IndexId ind metadata.put(Ccr.CCR_CUSTOM_METADATA_REMOTE_CLUSTER_NAME_KEY, remoteClusterAlias); imdBuilder.putCustom(Ccr.CCR_CUSTOM_METADATA_KEY, metadata); + imdBuilder.settings(leaderIndexMetaData.getSettings()); + + // Copy mappings from leader IMD to follow IMD + for (ObjectObjectCursor cursor : leaderIndexMetaData.getMappings()) { + imdBuilder.putMapping(cursor.value); + } + + imdBuilder.setRoutingNumShards(leaderIndexMetaData.getRoutingNumShards()); + // We assert that insync allocation ids are not empty in `PrimaryShardAllocator` + for (IntObjectCursor> entry : leaderIndexMetaData.getInSyncAllocationIds()) { + imdBuilder.putInSyncAllocationIds(entry.key, Collections.singleton(IN_SYNC_ALLOCATION_ID)); + } + return imdBuilder.build(); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPutFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPutFollowAction.java index 7b21422cb9867..d7a2edd21d26f 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPutFollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPutFollowAction.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.ccr.rest; +import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentParser; @@ -38,7 +39,8 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient static Request createRequest(RestRequest restRequest) throws IOException { try (XContentParser parser = restRequest.contentOrSourceParamParser()) { - return Request.fromXContent(parser, restRequest.param("index")); + ActiveShardCount waitForActiveShards = ActiveShardCount.parseString(restRequest.param("wait_for_active_shards")); + return Request.fromXContent(parser, restRequest.param("index"), waitForActiveShards); } } } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java index 65fd80325e716..4a6c5411737c6 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java @@ -18,6 +18,7 @@ import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.analysis.common.CommonAnalysisPlugin; import org.elasticsearch.client.Client; import org.elasticsearch.client.Requests; @@ -277,8 +278,13 @@ protected final ClusterHealthStatus ensureLeaderGreen(String... indices) { } protected final ClusterHealthStatus ensureFollowerGreen(String... indices) { + return ensureFollowerGreen(false, indices); + } + + protected final ClusterHealthStatus ensureFollowerGreen(boolean waitForNoInitializingShards, String... indices) { logger.info("ensure green follower indices {}", Arrays.toString(indices)); - return ensureColor(clusterGroup.followerCluster, ClusterHealthStatus.GREEN, TimeValue.timeValueSeconds(30), false, indices); + return ensureColor(clusterGroup.followerCluster, ClusterHealthStatus.GREEN, TimeValue.timeValueSeconds(30), + waitForNoInitializingShards, indices); } private ClusterHealthStatus ensureColor(TestCluster testCluster, @@ -411,10 +417,15 @@ protected String getIndexSettings(final int numberOfShards, final int numberOfRe } public static PutFollowAction.Request putFollow(String leaderIndex, String followerIndex) { + return putFollow(leaderIndex, followerIndex, ActiveShardCount.ONE); + } + + public static PutFollowAction.Request putFollow(String leaderIndex, String followerIndex, ActiveShardCount waitForActiveShards) { PutFollowAction.Request request = new PutFollowAction.Request(); request.setRemoteCluster("leader_cluster"); request.setLeaderIndex(leaderIndex); request.setFollowRequest(resumeFollow(followerIndex)); + request.waitForActiveShards(waitForActiveShards); return request; } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java index ad8f545fa9dc0..48531c7d28f9a 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack; import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; +import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; @@ -98,6 +99,7 @@ protected PutFollowAction.Request getPutFollowRequest(String leaderIndex, String request.setRemoteCluster("local"); request.setLeaderIndex(leaderIndex); request.setFollowRequest(getResumeFollowRequest(followerIndex)); + request.waitForActiveShards(ActiveShardCount.ONE); return request; } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java index 648c295efa817..dec671f6e6340 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java @@ -8,6 +8,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; @@ -30,7 +31,10 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.health.ClusterIndexHealth; +import org.elasticsearch.cluster.health.ClusterShardHealth; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.cluster.service.ClusterService; @@ -91,14 +95,12 @@ public class IndexFollowingIT extends CcrIntegTestCase { public void testFollowIndex() throws Exception { final int numberOfPrimaryShards = randomIntBetween(1, 3); - final String leaderIndexSettings = getIndexSettings(numberOfPrimaryShards, between(0, 1), + int numberOfReplicas = between(0, 1); + final String leaderIndexSettings = getIndexSettings(numberOfPrimaryShards, numberOfReplicas, singletonMap(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true")); assertAcked(leaderClient().admin().indices().prepareCreate("index1").setSource(leaderIndexSettings, XContentType.JSON)); ensureLeaderYellow("index1"); - final PutFollowAction.Request followRequest = putFollow("index1", "index2"); - followerClient().execute(PutFollowAction.INSTANCE, followRequest).get(); - final int firstBatchNumDocs = randomIntBetween(2, 64); logger.info("Indexing [{}] docs as first batch", firstBatchNumDocs); for (int i = 0; i < firstBatchNumDocs; i++) { @@ -106,6 +108,30 @@ public void testFollowIndex() throws Exception { leaderClient().prepareIndex("index1", "doc", Integer.toString(i)).setSource(source, XContentType.JSON).get(); } + boolean waitOnAll = randomBoolean(); + + final PutFollowAction.Request followRequest; + if (waitOnAll) { + followRequest = putFollow("index1", "index2", ActiveShardCount.ALL); + } else { + followRequest = putFollow("index1", "index2", ActiveShardCount.ONE); + } + PutFollowAction.Response response = followerClient().execute(PutFollowAction.INSTANCE, followRequest).get(); + assertTrue(response.isFollowIndexCreated()); + assertTrue(response.isFollowIndexShardsAcked()); + assertTrue(response.isIndexFollowingStarted()); + + ClusterHealthRequest healthRequest = Requests.clusterHealthRequest("index2").waitForNoRelocatingShards(true); + ClusterIndexHealth indexHealth = followerClient().admin().cluster().health(healthRequest).actionGet().getIndices().get("index2"); + for (ClusterShardHealth shardHealth : indexHealth.getShards().values()) { + if (waitOnAll) { + assertTrue(shardHealth.isPrimaryActive()); + assertEquals(1 + numberOfReplicas, shardHealth.getActiveShards()); + } else { + assertTrue(shardHealth.isPrimaryActive()); + } + } + final Map firstBatchNumDocsPerShard = new HashMap<>(); final ShardStats[] firstBatchShardStats = leaderClient().admin().indices().prepareStats("index1").get().getIndex("index1").getShards(); @@ -152,6 +178,119 @@ public void testFollowIndex() throws Exception { assertMaxSeqNoOfUpdatesIsTransferred(resolveLeaderIndex("index1"), resolveFollowerIndex("index2"), numberOfPrimaryShards); } + public void testFollowIndexWithConcurrentMappingChanges() throws Exception { + final int numberOfPrimaryShards = randomIntBetween(1, 3); + final String leaderIndexSettings = getIndexSettings(numberOfPrimaryShards, between(0, 1), + singletonMap(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true")); + assertAcked(leaderClient().admin().indices().prepareCreate("index1").setSource(leaderIndexSettings, XContentType.JSON)); + ensureLeaderYellow("index1"); + + final int firstBatchNumDocs = randomIntBetween(2, 64); + logger.info("Indexing [{}] docs as first batch", firstBatchNumDocs); + for (int i = 0; i < firstBatchNumDocs; i++) { + final String source = String.format(Locale.ROOT, "{\"f\":%d}", i); + leaderClient().prepareIndex("index1", "doc", Integer.toString(i)).setSource(source, XContentType.JSON).get(); + } + + AtomicBoolean isRunning = new AtomicBoolean(true); + + // Concurrently index new docs with mapping changes + Thread thread = new Thread(() -> { + int docID = 10000; + char[] chars = "abcdeghijklmnopqrstuvwxyz".toCharArray(); + for (char c : chars) { + if (isRunning.get() == false) { + break; + } + final String source; + long valueToPutInDoc = randomLongBetween(0, 50000); + if (randomBoolean()) { + source = String.format(Locale.ROOT, "{\"%c\":%d}", c, valueToPutInDoc); + } else { + source = String.format(Locale.ROOT, "{\"%c\":\"%d\"}", c, valueToPutInDoc); + } + for (int i = 1; i < 10; i++) { + if (isRunning.get() == false) { + break; + } + leaderClient().prepareIndex("index1", "doc", Long.toString(docID++)).setSource(source, XContentType.JSON).get(); + if (rarely()) { + leaderClient().admin().indices().prepareFlush("index1").setForce(true).get(); + } + } + leaderClient().admin().indices().prepareFlush("index1").setForce(true).setWaitIfOngoing(true).get(); + } + }); + thread.start(); + + final PutFollowAction.Request followRequest = putFollow("index1", "index2", ActiveShardCount.NONE); + followerClient().execute(PutFollowAction.INSTANCE, followRequest).get(); + + ensureFollowerGreen("index2"); + + for (int i = 0; i < firstBatchNumDocs; i++) { + assertBusy(assertExpectedDocumentRunnable(i)); + } + + final int secondBatchNumDocs = randomIntBetween(2, 64); + logger.info("Indexing [{}] docs as second batch", secondBatchNumDocs); + for (int i = firstBatchNumDocs; i < firstBatchNumDocs + secondBatchNumDocs; i++) { + final String source = String.format(Locale.ROOT, "{\"f\":%d}", i); + leaderClient().prepareIndex("index1", "doc", Integer.toString(i)).setSource(source, XContentType.JSON).get(); + } + + for (int i = firstBatchNumDocs; i < firstBatchNumDocs + secondBatchNumDocs; i++) { + assertBusy(assertExpectedDocumentRunnable(i)); + } + + isRunning.set(false); + thread.join(); + } + + public void testFollowIndexWithoutWaitForComplete() throws Exception { + final int numberOfPrimaryShards = randomIntBetween(1, 3); + final String leaderIndexSettings = getIndexSettings(numberOfPrimaryShards, between(0, 1), + singletonMap(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true")); + assertAcked(leaderClient().admin().indices().prepareCreate("index1").setSource(leaderIndexSettings, XContentType.JSON)); + ensureLeaderYellow("index1"); + + final int firstBatchNumDocs = randomIntBetween(2, 64); + logger.info("Indexing [{}] docs as first batch", firstBatchNumDocs); + for (int i = 0; i < firstBatchNumDocs; i++) { + final String source = String.format(Locale.ROOT, "{\"f\":%d}", i); + leaderClient().prepareIndex("index1", "doc", Integer.toString(i)).setSource(source, XContentType.JSON).get(); + } + + final PutFollowAction.Request followRequest = putFollow("index1", "index2", ActiveShardCount.NONE); + PutFollowAction.Response response = followerClient().execute(PutFollowAction.INSTANCE, followRequest).get(); + + assertTrue(response.isFollowIndexCreated()); + assertFalse(response.isFollowIndexShardsAcked()); + assertFalse(response.isIndexFollowingStarted()); + + // Check that the index exists, would throw index not found exception if the index is missing + followerClient().admin().indices().prepareGetIndex().addIndices("index2").get(); + ensureFollowerGreen(true, "index2"); + + final Map firstBatchNumDocsPerShard = new HashMap<>(); + final ShardStats[] firstBatchShardStats = + leaderClient().admin().indices().prepareStats("index1").get().getIndex("index1").getShards(); + for (final ShardStats shardStats : firstBatchShardStats) { + if (shardStats.getShardRouting().primary()) { + long value = shardStats.getStats().getIndexing().getTotal().getIndexCount() - 1; + firstBatchNumDocsPerShard.put(shardStats.getShardRouting().shardId(), value); + } + } + + assertBusy(assertTask(numberOfPrimaryShards, firstBatchNumDocsPerShard)); + + for (int i = 0; i < firstBatchNumDocs; i++) { + assertBusy(assertExpectedDocumentRunnable(i)); + } + assertTotalNumberOfOptimizedIndexing(resolveFollowerIndex("index2"), numberOfPrimaryShards, firstBatchNumDocs); + pauseFollow("index2"); + } + public void testSyncMappings() throws Exception { final String leaderIndexSettings = getIndexSettings(2, between(0, 1), singletonMap(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true")); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java index 2c50411971a1b..f50f17c9e296d 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java @@ -39,14 +39,14 @@ public void testFollowIndex() throws Exception { assertAcked(client().admin().indices().prepareCreate("leader").setSource(leaderIndexSettings, XContentType.JSON)); ensureGreen("leader"); - final PutFollowAction.Request followRequest = getPutFollowRequest("leader", "follower"); - client().execute(PutFollowAction.INSTANCE, followRequest).get(); - final long firstBatchNumDocs = randomIntBetween(2, 64); for (int i = 0; i < firstBatchNumDocs; i++) { client().prepareIndex("leader", "doc").setSource("{}", XContentType.JSON).get(); } + final PutFollowAction.Request followRequest = getPutFollowRequest("leader", "follower"); + client().execute(PutFollowAction.INSTANCE, followRequest).get(); + assertBusy(() -> { assertThat(client().prepareSearch("follower").get().getHits().getTotalHits().value, equalTo(firstBatchNumDocs)); }); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java index 726a1c9893a50..d32a773ebe218 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.ccr.action; +import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.AbstractSerializingTestCase; @@ -30,7 +31,7 @@ protected PutFollowAction.Request createTestInstance() { @Override protected PutFollowAction.Request doParseInstance(XContentParser parser) throws IOException { - return PutFollowAction.Request.fromXContent(parser, null); + return PutFollowAction.Request.fromXContent(parser, null, ActiveShardCount.DEFAULT); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java index 47240a6a1da10..6ca91bd7f0dad 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java @@ -6,10 +6,12 @@ package org.elasticsearch.xpack.core.ccr.action; +import org.elasticsearch.Version; import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.common.ParseField; @@ -28,10 +30,10 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.FOLLOWER_INDEX_FIELD; -import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_READ_REQUEST_OPERATION_COUNT; -import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_READ_REQUEST_SIZE; import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_OUTSTANDING_READ_REQUESTS; import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_OUTSTANDING_WRITE_REQUESTS; +import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_READ_REQUEST_OPERATION_COUNT; +import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_READ_REQUEST_SIZE; import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_RETRY_DELAY_FIELD; import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_WRITE_BUFFER_COUNT; import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_WRITE_BUFFER_SIZE; @@ -105,7 +107,8 @@ public static class Request extends AcknowledgedRequest implements Indi ObjectParser.ValueType.STRING); } - public static Request fromXContent(final XContentParser parser, final String followerIndex) throws IOException { + public static Request fromXContent(final XContentParser parser, final String followerIndex, ActiveShardCount waitForActiveShards) + throws IOException { Request request = PARSER.parse(parser, followerIndex); if (followerIndex != null) { if (request.getFollowRequest().getFollowerIndex() == null) { @@ -116,11 +119,13 @@ public static Request fromXContent(final XContentParser parser, final String fol } } } + request.waitForActiveShards(waitForActiveShards); return request; } private String remoteCluster; private String leaderIndex; + private ActiveShardCount waitForActiveShards = ActiveShardCount.NONE; private ResumeFollowAction.Request followRequest; public Request() { @@ -142,6 +147,27 @@ public void setLeaderIndex(String leaderIndex) { this.leaderIndex = leaderIndex; } + public ActiveShardCount waitForActiveShards() { + return waitForActiveShards; + } + + /** + * Sets the number of shard copies that should be active for follower index creation to + * return. Defaults to {@link ActiveShardCount#NONE}, which will not wait for any shards + * to be active. Set this value to {@link ActiveShardCount#DEFAULT} to wait for the primary + * shard to be active. Set this value to {@link ActiveShardCount#ALL} to wait for all shards + * (primary and all replicas) to be active before returning. + * + * @param waitForActiveShards number of active shard copies to wait on + */ + public void waitForActiveShards(ActiveShardCount waitForActiveShards) { + if (waitForActiveShards.equals(ActiveShardCount.DEFAULT)) { + this.waitForActiveShards = ActiveShardCount.NONE; + } else { + this.waitForActiveShards = waitForActiveShards; + } + } + public ResumeFollowAction.Request getFollowRequest() { return followRequest; } @@ -176,6 +202,10 @@ public Request(StreamInput in) throws IOException { super(in); remoteCluster = in.readString(); leaderIndex = in.readString(); + // TODO: Update after backport + if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + waitForActiveShards(ActiveShardCount.readFrom(in)); + } followRequest = new ResumeFollowAction.Request(in); } @@ -184,6 +214,10 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(remoteCluster); out.writeString(leaderIndex); + // TODO: Update after backport + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + waitForActiveShards.writeTo(out); + } followRequest.writeTo(out); } @@ -206,12 +240,23 @@ public boolean equals(Object o) { Request request = (Request) o; return Objects.equals(remoteCluster, request.remoteCluster) && Objects.equals(leaderIndex, request.leaderIndex) && + Objects.equals(waitForActiveShards, request.waitForActiveShards) && Objects.equals(followRequest, request.followRequest); } @Override public int hashCode() { - return Objects.hash(remoteCluster, leaderIndex, followRequest); + return Objects.hash(remoteCluster, leaderIndex, waitForActiveShards, followRequest); + } + + @Override + public String toString() { + return "PutFollowAction.Request{" + + "remoteCluster='" + remoteCluster + '\'' + + ", leaderIndex='" + leaderIndex + '\'' + + ", waitForActiveShards=" + waitForActiveShards + + ", followRequest=" + followRequest + + '}'; } } @@ -280,6 +325,15 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(followIndexCreated, followIndexShardsAcked, indexFollowingStarted); } + + @Override + public String toString() { + return "PutFollowAction.Response{" + + "followIndexCreated=" + followIndexCreated + + ", followIndexShardsAcked=" + followIndexShardsAcked + + ", indexFollowingStarted=" + indexFollowingStarted + + '}'; + } } } diff --git a/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java b/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java index 65baeb5f168c4..f8ffce9cd817a 100644 --- a/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java +++ b/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java @@ -61,6 +61,7 @@ public void testBasicCCRAndILMIntegration() throws Exception { // Policy with the same name must exist in follower cluster too: putILMPolicy(policyName, "50GB", null, TimeValue.timeValueHours(7*24)); followIndex(indexName, indexName); + ensureGreen(indexName); // Aliases are not copied from leader index, so we need to add that for the rollover action in follower cluster: client().performRequest(new Request("PUT", "/" + indexName + "/_alias/logs")); @@ -116,6 +117,7 @@ public void testCCRUnfollowDuringSnapshot() throws Exception { } else if ("follow".equals(targetCluster)) { createNewSingletonPolicy("unfollow-only", "hot", new UnfollowAction(), TimeValue.ZERO); followIndex(indexName, indexName); + ensureGreen(indexName); // Create the repository before taking the snapshot. Request request = new Request("PUT", "/_snapshot/repo"); @@ -210,7 +212,7 @@ public void testCcrAndIlmWithRollover() throws Exception { "\"mappings\": {\"_doc\": {\"properties\": {\"field\": {\"type\": \"keyword\"}}}}, " + "\"aliases\": {\"" + alias + "\": {\"is_write_index\": true}} }"); assertOK(leaderClient.performRequest(createIndexRequest)); - // Check that the new index is creeg + // Check that the new index is created Request checkIndexRequest = new Request("GET", "/_cluster/health/" + indexName); checkIndexRequest.addParameter("wait_for_status", "green"); checkIndexRequest.addParameter("timeout", "70s"); @@ -226,6 +228,7 @@ public void testCcrAndIlmWithRollover() throws Exception { index(leaderClient, indexName, "1"); assertDocumentExists(leaderClient, indexName, "1"); + ensureGreen(indexName); assertBusy(() -> { assertDocumentExists(client(), indexName, "1"); // Sanity check that following_index setting has been set, so that we can verify later that this setting has been unset: diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.follow.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.follow.json index 635a4e62683bb..588dd60261252 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.follow.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.follow.json @@ -11,6 +11,13 @@ "required": true, "description": "The name of the follower index" } + }, + "params": { + "wait_for_active_shards": { + "type" : "string", + "description" : "Sets the number of shard copies that must be active before returning. Defaults to 0. Set to `all` for all shard copies, otherwise set to any non-negative value less than or equal to the total number of copies for the shard (number of replicas + 1)", + "default": "0" + } } }, "body": { From 9ca26b7e63b4f75f82842c432377ccee5cdc1e6f Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Tue, 29 Jan 2019 10:51:07 -0800 Subject: [PATCH 022/100] Remove more references to type in docs. (#37946) * Update the top-level 'getting started' guide. * Remove custom types from the painless getting started documentation. * Fix an incorrect references to '_doc' in the cardinality query docs. * Update the _update docs to use the typeless API format. --- docs/painless/painless-debugging.asciidoc | 10 ++++---- .../painless-getting-started.asciidoc | 6 ++--- .../metrics/cardinality-aggregation.asciidoc | 4 ++-- docs/reference/docs/update.asciidoc | 24 +++++++++---------- docs/reference/getting-started.asciidoc | 14 +++++------ 5 files changed, 28 insertions(+), 30 deletions(-) diff --git a/docs/painless/painless-debugging.asciidoc b/docs/painless/painless-debugging.asciidoc index c141cbc53252a..e335bfe74ebd7 100644 --- a/docs/painless/painless-debugging.asciidoc +++ b/docs/painless/painless-debugging.asciidoc @@ -18,10 +18,10 @@ context available to a {ref}/query-dsl-script-query.html[script query]. [source,js] --------------------------------------------------------- -PUT /hockey/player/1?refresh +PUT /hockey/_doc/1?refresh {"first":"johnny","last":"gaudreau","goals":[9,27,1],"assists":[17,46,0],"gp":[26,82,1]} -POST /hockey/player/1/_explain +POST /hockey/_explain/1 { "query": { "script": { @@ -31,7 +31,7 @@ POST /hockey/player/1/_explain } --------------------------------------------------------- // CONSOLE -// TEST[s/_explain/_explain?error_trace=false/ catch:/painless_explain_error/] +// TEST[s/_explain\/1/_explain\/1?error_trace=false/ catch:/painless_explain_error/] // The test system sends error_trace=true by default for easier debugging so // we have to override it to get a normal shaped response @@ -58,13 +58,13 @@ in the `_update` API: [source,js] --------------------------------------------------------- -POST /hockey/player/1/_update +POST /hockey/_update/1 { "script": "Debug.explain(ctx._source)" } --------------------------------------------------------- // CONSOLE -// TEST[continued s/_update/_update?error_trace=false/ catch:/painless_explain_error/] +// TEST[continued s/_update\/1/_update\/1?error_trace=false/ catch:/painless_explain_error/] The response looks like: diff --git a/docs/painless/painless-getting-started.asciidoc b/docs/painless/painless-getting-started.asciidoc index 936bd8e198c72..f562033471e31 100644 --- a/docs/painless/painless-getting-started.asciidoc +++ b/docs/painless/painless-getting-started.asciidoc @@ -10,7 +10,7 @@ To illustrate how Painless works, let's load some hockey stats into an Elasticse [source,js] ---------------------------------------------------------------- -PUT hockey/player/_bulk?refresh +PUT hockey/_bulk?refresh {"index":{"_id":1}} {"first":"johnny","last":"gaudreau","goals":[9,27,1],"assists":[17,46,0],"gp":[26,82,1],"born":"1993/08/13"} {"index":{"_id":2}} @@ -158,7 +158,7 @@ To change player 1's last name to `hockey`, simply set `ctx._source.last` to the [source,js] ---------------------------------------------------------------- -POST hockey/player/1/_update +POST hockey/_update/1 { "script": { "lang": "painless", @@ -176,7 +176,7 @@ the player's nickname, _hockey_. [source,js] ---------------------------------------------------------------- -POST hockey/player/1/_update +POST hockey/_update/1 { "script": { "lang": "painless", diff --git a/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc b/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc index a451c6da0db95..405a9d89107f2 100644 --- a/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc @@ -49,7 +49,7 @@ POST /sales/_search?size=0 "aggs" : { "type_count" : { "cardinality" : { - "field" : "_doc", + "field" : "type", "precision_threshold": 100 <1> } } @@ -214,7 +214,7 @@ POST /sales/_search?size=0 "script" : { "id": "my_script", "params": { - "type_field": "_doc", + "type_field": "type", "promoted_field": "promoted" } } diff --git a/docs/reference/docs/update.asciidoc b/docs/reference/docs/update.asciidoc index 42840b1b0a5ec..64c0f67bc722c 100644 --- a/docs/reference/docs/update.asciidoc +++ b/docs/reference/docs/update.asciidoc @@ -32,7 +32,7 @@ Now, we can execute a script that would increment the counter: [source,js] -------------------------------------------------- -POST test/_doc/1/_update +POST test/_update/1 { "script" : { "source": "ctx._source.counter += params.count", @@ -51,7 +51,7 @@ will still add it, since it's a list): [source,js] -------------------------------------------------- -POST test/_doc/1/_update +POST test/_update/1 { "script" : { "source": "ctx._source.tags.add(params.tag)", @@ -73,7 +73,7 @@ remove only one occurrence of it: [source,js] -------------------------------------------------- -POST test/_doc/1/_update +POST test/_update/1 { "script" : { "source": "if (ctx._source.tags.contains(params.tag)) { ctx._source.tags.remove(ctx._source.tags.indexOf(params.tag)) }", @@ -95,7 +95,7 @@ We can also add a new field to the document: [source,js] -------------------------------------------------- -POST test/_doc/1/_update +POST test/_update/1 { "script" : "ctx._source.new_field = 'value_of_new_field'" } @@ -107,7 +107,7 @@ Or remove a field from the document: [source,js] -------------------------------------------------- -POST test/_doc/1/_update +POST test/_update/1 { "script" : "ctx._source.remove('new_field')" } @@ -121,7 +121,7 @@ the doc if the `tags` field contain `green`, otherwise it does nothing [source,js] -------------------------------------------------- -POST test/_doc/1/_update +POST test/_update/1 { "script" : { "source": "if (ctx._source.tags.contains(params.tag)) { ctx.op = 'delete' } else { ctx.op = 'none' }", @@ -148,7 +148,7 @@ existing document: [source,js] -------------------------------------------------- -POST test/_doc/1/_update +POST test/_update/1 { "doc" : { "name" : "new_name" @@ -169,7 +169,7 @@ By default updates that don't change anything detect that they don't change anyt [source,js] -------------------------------------------------- -POST test/_doc/1/_update +POST test/_update/1 { "doc" : { "name" : "new_name" @@ -204,7 +204,7 @@ You can disable this behavior by setting "detect_noop": false like this: [source,js] -------------------------------------------------- -POST test/_doc/1/_update +POST test/_update/1 { "doc" : { "name" : "new_name" @@ -225,7 +225,7 @@ will be inserted as a new document. If the document does exist, then the [source,js] -------------------------------------------------- -POST test/_doc/1/_update +POST test/_update/1 { "script" : { "source": "ctx._source.counter += params.count", @@ -251,7 +251,7 @@ or not -- i.e. the script handles initializing the document instead of the [source,js] -------------------------------------------------- -POST sessions/session/dh3sgudg8gsrgl/_update +POST sessions/_update/dh3sgudg8gsrgl { "scripted_upsert":true, "script" : { @@ -280,7 +280,7 @@ value: [source,js] -------------------------------------------------- -POST test/_doc/1/_update +POST test/_update/1 { "doc" : { "name" : "new_name" diff --git a/docs/reference/getting-started.asciidoc b/docs/reference/getting-started.asciidoc index d8656f7ac4c25..bee2ae5194477 100755 --- a/docs/reference/getting-started.asciidoc +++ b/docs/reference/getting-started.asciidoc @@ -65,9 +65,7 @@ A type used to be a logical category/partition of your index to allow you to sto [float] === Document -A document is a basic unit of information that can be indexed. For example, you can have a document for a single customer, another document for a single product, and yet another for a single order. This document is expressed in http://json.org/[JSON] (JavaScript Object Notation) which is a ubiquitous internet data interchange format. - -Within an index/type, you can store as many documents as you want. Note that although a document physically resides in an index, a document actually must be indexed/assigned to a type inside an index. +A document is a basic unit of information that can be indexed. For example, you can have a document for a single customer, another document for a single product, and yet another for a single order. This document is expressed in http://json.org/[JSON] (JavaScript Object Notation) which is a ubiquitous internet data interchange format. Within an index, you can store as many documents as you want. [[getting-started-shards-and-replicas]] [float] @@ -496,7 +494,7 @@ If we study the above commands carefully, we can actually see a pattern of how w [source,js] -------------------------------------------------- - /// + /// -------------------------------------------------- // NOTCONSOLE @@ -572,7 +570,7 @@ This example shows how to update our previous document (ID of 1) by changing the [source,js] -------------------------------------------------- -POST /customer/_doc/1/_update?pretty +POST /customer/_update/1?pretty { "doc": { "name": "Jane Doe" } } @@ -584,7 +582,7 @@ This example shows how to update our previous document (ID of 1) by changing the [source,js] -------------------------------------------------- -POST /customer/_doc/1/_update?pretty +POST /customer/_update/1?pretty { "doc": { "name": "Jane Doe", "age": 20 } } @@ -596,7 +594,7 @@ Updates can also be performed by using simple scripts. This example uses a scrip [source,js] -------------------------------------------------- -POST /customer/_doc/1/_update?pretty +POST /customer/_update/1?pretty { "script" : "ctx._source.age += 5" } @@ -719,7 +717,7 @@ yellow open bank l7sSYV2cQXmu6_4rJWVIww 5 1 1000 0 12 // TESTRESPONSE[s/128.6kb/\\d+(\\.\\d+)?[mk]?b/] // TESTRESPONSE[s/l7sSYV2cQXmu6_4rJWVIww/.+/ _cat] -Which means that we just successfully bulk indexed 1000 documents into the bank index (under the `_doc` type). +Which means that we just successfully bulk indexed 1000 documents into the bank index. [[getting-started-search-API]] === The Search API From 34d61d3231261794a58a918fce9642cf6699fbab Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Tue, 29 Jan 2019 12:51:34 -0600 Subject: [PATCH 023/100] ML: ignore unknown fields for JobTaskState (#37982) --- .../elasticsearch/xpack/core/ml/job/config/JobTaskState.java | 3 +-- .../elasticsearch/xpack/ml/job/config/JobTaskStateTests.java | 5 +++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobTaskState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobTaskState.java index d979b897ad43a..08f73d791e53f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobTaskState.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobTaskState.java @@ -29,8 +29,7 @@ public class JobTaskState implements PersistentTaskState { private static ParseField ALLOCATION_ID = new ParseField("allocation_id"); private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>(NAME, - args -> new JobTaskState((JobState) args[0], (Long) args[1])); + new ConstructingObjectParser<>(NAME, true, args -> new JobTaskState((JobState) args[0], (Long) args[1])); static { PARSER.declareField(constructorArg(), p -> { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobTaskStateTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobTaskStateTests.java index 4dfd1965804e5..26560f1034f9e 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobTaskStateTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobTaskStateTests.java @@ -27,4 +27,9 @@ protected Writeable.Reader instanceReader() { protected JobTaskState doParseInstance(XContentParser parser) { return JobTaskState.fromXContent(parser); } + + @Override + protected boolean supportsUnknownFields() { + return true; + } } From 697b2fbe520942a2212e1778b5b251138db93425 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Tue, 29 Jan 2019 21:10:03 +0200 Subject: [PATCH 024/100] Remove implicit index monitor privilege (#37774) Restricted indices (currently only .security-6 and .security) are special internal indices that require setting the `allow_restricted_indices` flag on every index permission that covers them. If this flag is `false` (default) the permission will not cover these and actions against them will not be authorized. However, the monitoring APIs were the only exception to this rule. This exception is herein forfeited and index monitoring privileges have to be granted explicitly, using the `allow_restricted_indices` flag on the permission, as is the case for any other index privilege. --- .../migration/migrate_7_0/api.asciidoc | 12 ++ .../authz/permission/IndicesPermission.java | 8 +- .../authz/store/ReservedRolesStoreTests.java | 25 ++++ .../MultipleIndicesPermissionsTests.java | 108 +++++++++++++- .../test/SecuritySettingsSource.java | 1 + .../authz/AuthorizationServiceTests.java | 141 +++++++++++------- 6 files changed, 230 insertions(+), 65 deletions(-) diff --git a/docs/reference/migration/migrate_7_0/api.asciidoc b/docs/reference/migration/migrate_7_0/api.asciidoc index 3972be43685cc..bb151edb778e2 100644 --- a/docs/reference/migration/migrate_7_0/api.asciidoc +++ b/docs/reference/migration/migrate_7_0/api.asciidoc @@ -126,3 +126,15 @@ removed. The `_termvector` endpoint was deprecated in 2.0 and has now been removed. The endpoint `_termvectors` (plural) should be used instead. + +[float] +==== When {security-features} are enabled, index monitoring APIs over restricted indices are not authorized implicitly anymore + +Restricted indices (currently only `.security-6` and `.security`) are special internal +indices that require setting the `allow_restricted_indices` flag on every index +permission that covers them. If this flag is `false` (default) the permission +will not cover these and actions against them will not be authorized. +However, the monitoring APIs were the only exception to this rule. This exception +has been forfeited and index monitoring privileges have to be granted explicitly, +using the `allow_restricted_indices` flag on the permission (as any other index +privilege). diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java index 4c2a479721a2a..27fa8b2cd9da0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java @@ -259,12 +259,8 @@ private boolean check(String action) { private boolean check(String action, String index) { assert index != null; - return check(action) && (indexNameMatcher.test(index) - && (allowRestrictedIndices - // all good if it is not restricted - || (false == RestrictedIndicesNames.NAMES_SET.contains(index)) - // allow monitor as a special case, even for restricted - || IndexPrivilege.MONITOR.predicate().test(action))); + return check(action) && indexNameMatcher.test(index) + && (allowRestrictedIndices || (false == RestrictedIndicesNames.NAMES_SET.contains(index))); } boolean hasQuery() { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java index 8711a6c318e58..35e2043acd809 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java @@ -144,6 +144,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.Map; import static org.hamcrest.Matchers.hasEntry; @@ -563,9 +564,33 @@ public void testRemoteMonitoringCollectorRole() { assertThat(remoteMonitoringAgentRole.indices().allowedIndicesMatcher(IndexAction.NAME) .test(randomFrom(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX, RestrictedIndicesNames.SECURITY_INDEX_NAME)), is(false)); + assertMonitoringOnRestrictedIndices(remoteMonitoringAgentRole); + assertNoAccessAllowed(remoteMonitoringAgentRole, RestrictedIndicesNames.NAMES_SET); } + private void assertMonitoringOnRestrictedIndices(Role role) { + final Settings indexSettings = Settings.builder().put("index.version.created", Version.CURRENT).build(); + final MetaData metaData = new MetaData.Builder() + .put(new IndexMetaData.Builder(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX) + .settings(indexSettings) + .numberOfShards(1) + .numberOfReplicas(0) + .putAlias(new AliasMetaData.Builder(RestrictedIndicesNames.SECURITY_INDEX_NAME).build()) + .build(), true) + .build(); + final FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY); + final List indexMonitoringActionNamesList = Arrays.asList(IndicesStatsAction.NAME, IndicesSegmentsAction.NAME, + GetSettingsAction.NAME, IndicesShardStoresAction.NAME, UpgradeStatusAction.NAME, RecoveryAction.NAME); + for (final String indexMonitoringActionName : indexMonitoringActionNamesList) { + final Map authzMap = role.indices().authorize(indexMonitoringActionName, + Sets.newHashSet(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX, RestrictedIndicesNames.SECURITY_INDEX_NAME), metaData, + fieldPermissionsCache); + assertThat(authzMap.get(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX).isGranted(), is(true)); + assertThat(authzMap.get(RestrictedIndicesNames.SECURITY_INDEX_NAME).isGranted(), is(true)); + } + } + public void testReportingUserRole() { final TransportRequest request = mock(TransportRequest.class); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java index 675300e25760e..da03e9ffe3d1e 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java @@ -7,14 +7,27 @@ import org.elasticsearch.ElasticsearchSecurityException; import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; +import org.elasticsearch.action.admin.indices.segments.IndicesSegmentResponse; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.action.admin.indices.upgrade.get.UpgradeStatusResponse; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.MultiSearchResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.SecurityIntegTestCase; import org.elasticsearch.test.SecuritySettingsSource; +import org.elasticsearch.xpack.core.security.authc.support.Hasher; +import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; +import org.junit.After; +import org.junit.Before; import java.util.Collections; @@ -25,11 +38,29 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.containsInAnyOrder; public class MultipleIndicesPermissionsTests extends SecurityIntegTestCase { protected static final SecureString USERS_PASSWD = new SecureString("passwd".toCharArray()); + @Before + public void waitForSecurityIndexWritable() throws Exception { + // adds a dummy user to the native realm to force .security index creation + securityClient().preparePutUser("dummy_user", "password".toCharArray(), Hasher.BCRYPT, "missing_role").get(); + assertSecurityIndexActive(); + } + + @After + public void cleanupSecurityIndex() throws Exception { + super.deleteSecurityIndex(); + } + + @Override + protected boolean addMockHttpTransport() { + return false; // enable http + } + @Override protected String configRoles() { return SecuritySettingsSource.TEST_ROLE + ":\n" + @@ -49,6 +80,12 @@ protected String configRoles() { " - names: 'a'\n" + " privileges: [all]\n" + "\n" + + "role_monitor_all_unrestricted_indices:\n" + + " cluster: [monitor]\n" + + " indices:\n" + + " - names: '*'\n" + + " privileges: [monitor]\n" + + "\n" + "role_b:\n" + " indices:\n" + " - names: 'b'\n" + @@ -60,14 +97,16 @@ protected String configUsers() { final String usersPasswdHashed = new String(getFastStoredHashAlgoForTests().hash(USERS_PASSWD)); return SecuritySettingsSource.CONFIG_STANDARD_USER + "user_a:" + usersPasswdHashed + "\n" + - "user_ab:" + usersPasswdHashed + "\n"; + "user_ab:" + usersPasswdHashed + "\n" + + "user_monitor:" + usersPasswdHashed + "\n"; } @Override protected String configUsersRoles() { return SecuritySettingsSource.CONFIG_STANDARD_USER_ROLES + "role_a:user_a,user_ab\n" + - "role_b:user_ab\n"; + "role_b:user_ab\n" + + "role_monitor_all_unrestricted_indices:user_monitor\n"; } public void testSingleRole() throws Exception { @@ -127,6 +166,71 @@ public void testSingleRole() throws Exception { assertHitCount(searchResponse, 1); } + public void testMonitorRestrictedWildcards() throws Exception { + + IndexResponse indexResponse = index("foo", "type", jsonBuilder() + .startObject() + .field("name", "value") + .endObject()); + assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult()); + + indexResponse = index("foobar", "type", jsonBuilder() + .startObject() + .field("name", "value") + .endObject()); + assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult()); + + indexResponse = index("foobarfoo", "type", jsonBuilder() + .startObject() + .field("name", "value") + .endObject()); + assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult()); + + refresh(); + + final Client client = client() + .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user_monitor", USERS_PASSWD))); + + final GetSettingsResponse getSettingsResponse = client.admin().indices().prepareGetSettings(randomFrom("*", "_all", "foo*")).get(); + assertThat(getSettingsResponse.getIndexToSettings().size(), is(3)); + assertThat(getSettingsResponse.getIndexToSettings().containsKey("foo"), is(true)); + assertThat(getSettingsResponse.getIndexToSettings().containsKey("foobar"), is(true)); + assertThat(getSettingsResponse.getIndexToSettings().containsKey("foobarfoo"), is(true)); + + final IndicesShardStoresResponse indicesShardsStoresResponse = client.admin().indices() + .prepareShardStores(randomFrom("*", "_all", "foo*")).setShardStatuses("all").get(); + assertThat(indicesShardsStoresResponse.getStoreStatuses().size(), is(3)); + assertThat(indicesShardsStoresResponse.getStoreStatuses().containsKey("foo"), is(true)); + assertThat(indicesShardsStoresResponse.getStoreStatuses().containsKey("foobar"), is(true)); + assertThat(indicesShardsStoresResponse.getStoreStatuses().containsKey("foobarfoo"), is(true)); + + final UpgradeStatusResponse upgradeStatusResponse = client.admin().indices().prepareUpgradeStatus(randomFrom("*", "_all", "foo*")) + .get(); + assertThat(upgradeStatusResponse.getIndices().size(), is(3)); + assertThat(upgradeStatusResponse.getIndices().keySet(), containsInAnyOrder("foo", "foobar", "foobarfoo")); + + final IndicesStatsResponse indicesStatsResponse = client.admin().indices().prepareStats(randomFrom("*", "_all", "foo*")).get(); + assertThat(indicesStatsResponse.getIndices().size(), is(3)); + assertThat(indicesStatsResponse.getIndices().keySet(), containsInAnyOrder("foo", "foobar", "foobarfoo")); + + final IndicesSegmentResponse indicesSegmentResponse = client.admin().indices().prepareSegments("*").get(); + assertThat(indicesSegmentResponse.getIndices().size(), is(3)); + assertThat(indicesSegmentResponse.getIndices().keySet(), containsInAnyOrder("foo", "foobar", "foobarfoo")); + + final RecoveryResponse indicesRecoveryResponse = client.admin().indices().prepareRecoveries("*").get(); + assertThat(indicesRecoveryResponse.shardRecoveryStates().size(), is(3)); + assertThat(indicesRecoveryResponse.shardRecoveryStates().keySet(), containsInAnyOrder("foo", "foobar", "foobarfoo")); + + // test _cat/indices with wildcards that cover unauthorized indices (".security" in this case) + RequestOptions.Builder optionsBuilder = RequestOptions.DEFAULT.toBuilder(); + optionsBuilder.addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue("user_monitor", USERS_PASSWD)); + RequestOptions options = optionsBuilder.build(); + Request catIndicesRequest = new Request("GET", "/_cat/indices/" + randomFrom("*", "_all", "foo*")); + catIndicesRequest.setOptions(options); + Response catIndicesResponse = getRestClient().performRequest(catIndicesRequest); + assertThat(catIndicesResponse.getStatusLine().getStatusCode() < 300, is(true)); + } + public void testMultipleRoles() throws Exception { IndexResponse indexResponse = index("a", "type", jsonBuilder() .startObject() diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java index adeb4a7f86569..d72130db2f571 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java @@ -78,6 +78,7 @@ public class SecuritySettingsSource extends NodeConfigurationSource { " cluster: [ ALL ]\n" + " indices:\n" + " - names: '*'\n" + + " allow_restricted_indices: true\n" + " privileges: [ ALL ]\n"; private final Path parentFolder; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java index dccc8f3ce587a..171e11614c5f3 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java @@ -73,6 +73,7 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -151,7 +152,8 @@ import static org.elasticsearch.test.SecurityTestsUtils.assertThrowsAuthorizationException; import static org.elasticsearch.test.SecurityTestsUtils.assertThrowsAuthorizationExceptionRunAs; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; -import static org.hamcrest.Matchers.arrayContaining; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_SECURITY_INDEX; +import static org.hamcrest.Matchers.arrayContainingInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; @@ -804,7 +806,7 @@ public void testRunAsRequestWithValidPermissions() { verifyNoMoreInteractions(auditTrail); } - public void testNonXPackUserCannotExecuteOperationAgainstSecurityIndex() { + public void testGrantAllRestrictedUserCannotExecuteOperationAgainstSecurityIndices() { RoleDescriptor role = new RoleDescriptor("all access", new String[]{"all"}, new IndicesPrivileges[]{IndicesPrivileges.builder().indices("*").privileges("all").build()}, null); final Authentication authentication = createAuthentication(new User("all_access_user", "all_access")); @@ -812,29 +814,41 @@ public void testNonXPackUserCannotExecuteOperationAgainstSecurityIndex() { ClusterState state = mock(ClusterState.class); when(clusterService.state()).thenReturn(state); when(state.metaData()).thenReturn(MetaData.builder() - .put(new IndexMetaData.Builder(SECURITY_INDEX_NAME) - .settings(Settings.builder().put("index.version.created", Version.CURRENT).build()) - .numberOfShards(1).numberOfReplicas(0).build(), true) + .put(new IndexMetaData.Builder(INTERNAL_SECURITY_INDEX) + .putAlias(new AliasMetaData.Builder(SECURITY_INDEX_NAME).build()) + .settings(Settings.builder().put("index.version.created", Version.CURRENT).build()) + .numberOfShards(1) + .numberOfReplicas(0) + .build(),true) .build()); final String requestId = AuditUtil.getOrGenerateRequestId(threadContext); List> requests = new ArrayList<>(); - requests.add(new Tuple<>(BulkAction.NAME + "[s]", - new DeleteRequest(SECURITY_INDEX_NAME, "type", "id"))); - requests.add(new Tuple<>(UpdateAction.NAME, - new UpdateRequest(SECURITY_INDEX_NAME, "type", "id"))); - requests.add(new Tuple<>(BulkAction.NAME + "[s]", - new IndexRequest(SECURITY_INDEX_NAME, "type", "id"))); - requests.add(new Tuple<>(SearchAction.NAME, new SearchRequest(SECURITY_INDEX_NAME))); - requests.add(new Tuple<>(TermVectorsAction.NAME, - new TermVectorsRequest(SECURITY_INDEX_NAME, "type", "id"))); - requests.add(new Tuple<>(GetAction.NAME, new GetRequest(SECURITY_INDEX_NAME, "type", "id"))); + requests.add( + new Tuple<>(BulkAction.NAME + "[s]", new DeleteRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), "id"))); + requests.add(new Tuple<>(UpdateAction.NAME, new UpdateRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), "id"))); + requests.add(new Tuple<>(BulkAction.NAME + "[s]", new IndexRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); + requests.add(new Tuple<>(SearchAction.NAME, new SearchRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); requests.add(new Tuple<>(TermVectorsAction.NAME, - new TermVectorsRequest(SECURITY_INDEX_NAME, "type", "id"))); + new TermVectorsRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), "type", "id"))); + requests.add(new Tuple<>(GetAction.NAME, new GetRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), "id"))); requests.add(new Tuple<>(IndicesAliasesAction.NAME, new IndicesAliasesRequest() - .addAliasAction(AliasActions.add().alias("security_alias").index(SECURITY_INDEX_NAME)))); + .addAliasAction(AliasActions.add().alias("security_alias").index(INTERNAL_SECURITY_INDEX)))); + requests.add(new Tuple<>(UpdateSettingsAction.NAME, + new UpdateSettingsRequest().indices(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); + // cannot execute monitor operations + requests.add(new Tuple<>(IndicesStatsAction.NAME, + new IndicesStatsRequest().indices(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); requests.add( - new Tuple<>(UpdateSettingsAction.NAME, new UpdateSettingsRequest().indices(SECURITY_INDEX_NAME))); + new Tuple<>(RecoveryAction.NAME, new RecoveryRequest().indices(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); + requests.add(new Tuple<>(IndicesSegmentsAction.NAME, + new IndicesSegmentsRequest().indices(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); + requests.add(new Tuple<>(GetSettingsAction.NAME, + new GetSettingsRequest().indices(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); + requests.add(new Tuple<>(IndicesShardStoresAction.NAME, + new IndicesShardStoresRequest().indices(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); + requests.add(new Tuple<>(UpgradeStatusAction.NAME, + new UpgradeStatusRequest().indices(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); for (Tuple requestTuple : requests) { String action = requestTuple.v1(); @@ -847,12 +861,12 @@ public void testNonXPackUserCannotExecuteOperationAgainstSecurityIndex() { } // we should allow waiting for the health of the index or any index if the user has this permission - ClusterHealthRequest request = new ClusterHealthRequest(SECURITY_INDEX_NAME); + ClusterHealthRequest request = new ClusterHealthRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)); authorize(authentication, ClusterHealthAction.NAME, request); verify(auditTrail).accessGranted(requestId, authentication, ClusterHealthAction.NAME, request, new String[]{role.getName()}); // multiple indices - request = new ClusterHealthRequest(SECURITY_INDEX_NAME, "foo", "bar"); + request = new ClusterHealthRequest(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX, "foo", "bar"); authorize(authentication, ClusterHealthAction.NAME, request); verify(auditTrail).accessGranted(requestId, authentication, ClusterHealthAction.NAME, request, new String[]{role.getName()}); verifyNoMoreInteractions(auditTrail); @@ -863,17 +877,24 @@ public void testNonXPackUserCannotExecuteOperationAgainstSecurityIndex() { assertEquals(IndicesAndAliasesResolver.NO_INDICES_OR_ALIASES_LIST, Arrays.asList(searchRequest.indices())); } - public void testGrantedNonXPackUserCanExecuteMonitoringOperationsAgainstSecurityIndex() { - RoleDescriptor role = new RoleDescriptor("all access", new String[]{"all"}, - new IndicesPrivileges[]{IndicesPrivileges.builder().indices("*").privileges("all").build()}, null); - final Authentication authentication = createAuthentication(new User("all_access_user", "all_access")); - roleMap.put("all_access", role); + public void testMonitoringOperationsAgainstSecurityIndexRequireAllowRestricted() { + final RoleDescriptor restrictedMonitorRole = new RoleDescriptor("restricted_monitor", null, + new IndicesPrivileges[] { IndicesPrivileges.builder().indices("*").privileges("monitor").build() }, null); + final RoleDescriptor unrestrictedMonitorRole = new RoleDescriptor("unrestricted_monitor", null, new IndicesPrivileges[] { + IndicesPrivileges.builder().indices("*").privileges("monitor").allowRestrictedIndices(true).build() }, null); + roleMap.put("restricted_monitor", restrictedMonitorRole); + roleMap.put("unrestricted_monitor", unrestrictedMonitorRole); + final Authentication restrictedUserAuthn = createAuthentication(new User("restricted_user", "restricted_monitor")); + final Authentication unrestrictedUserAuthn = createAuthentication(new User("unrestricted_user", "unrestricted_monitor")); ClusterState state = mock(ClusterState.class); when(clusterService.state()).thenReturn(state); when(state.metaData()).thenReturn(MetaData.builder() - .put(new IndexMetaData.Builder(SECURITY_INDEX_NAME) - .settings(Settings.builder().put("index.version.created", Version.CURRENT).build()) - .numberOfShards(1).numberOfReplicas(0).build(), true) + .put(new IndexMetaData.Builder(INTERNAL_SECURITY_INDEX) + .putAlias(new AliasMetaData.Builder(SECURITY_INDEX_NAME).build()) + .settings(Settings.builder().put("index.version.created", Version.CURRENT).build()) + .numberOfShards(1) + .numberOfReplicas(0) + .build(), true) .build()); final String requestId = AuditUtil.getOrGenerateRequestId(threadContext); @@ -882,16 +903,18 @@ public void testGrantedNonXPackUserCanExecuteMonitoringOperationsAgainstSecurity requests.add(new Tuple<>(RecoveryAction.NAME, new RecoveryRequest().indices(SECURITY_INDEX_NAME))); requests.add(new Tuple<>(IndicesSegmentsAction.NAME, new IndicesSegmentsRequest().indices(SECURITY_INDEX_NAME))); requests.add(new Tuple<>(GetSettingsAction.NAME, new GetSettingsRequest().indices(SECURITY_INDEX_NAME))); - requests.add(new Tuple<>(IndicesShardStoresAction.NAME, - new IndicesShardStoresRequest().indices(SECURITY_INDEX_NAME))); - requests.add(new Tuple<>(UpgradeStatusAction.NAME, - new UpgradeStatusRequest().indices(SECURITY_INDEX_NAME))); + requests.add(new Tuple<>(IndicesShardStoresAction.NAME, new IndicesShardStoresRequest().indices(SECURITY_INDEX_NAME))); + requests.add(new Tuple<>(UpgradeStatusAction.NAME, new UpgradeStatusRequest().indices(SECURITY_INDEX_NAME))); for (final Tuple requestTuple : requests) { final String action = requestTuple.v1(); final TransportRequest request = requestTuple.v2(); - authorize(authentication, action, request); - verify(auditTrail).accessGranted(requestId, authentication, action, request, new String[]{role.getName()}); + assertThrowsAuthorizationException(() -> authorize(restrictedUserAuthn, action, request), action, "restricted_user"); + verify(auditTrail).accessDenied(requestId, restrictedUserAuthn, action, request, new String[] { "restricted_monitor" }); + verifyNoMoreInteractions(auditTrail); + authorize(unrestrictedUserAuthn, action, request); + verify(auditTrail).accessGranted(requestId, unrestrictedUserAuthn, action, request, new String[] { "unrestricted_monitor" }); + verifyNoMoreInteractions(auditTrail); } } @@ -901,34 +924,35 @@ public void testSuperusersCanExecuteOperationAgainstSecurityIndex() { ClusterState state = mock(ClusterState.class); when(clusterService.state()).thenReturn(state); when(state.metaData()).thenReturn(MetaData.builder() - .put(new IndexMetaData.Builder(SECURITY_INDEX_NAME) - .settings(Settings.builder().put("index.version.created", Version.CURRENT).build()) - .numberOfShards(1).numberOfReplicas(0).build(), true) + .put(new IndexMetaData.Builder(INTERNAL_SECURITY_INDEX) + .putAlias(new AliasMetaData.Builder(SECURITY_INDEX_NAME).build()) + .settings(Settings.builder().put("index.version.created", Version.CURRENT).build()) + .numberOfShards(1) + .numberOfReplicas(0) + .build(), true) .build()); final String requestId = AuditUtil.getOrGenerateRequestId(threadContext); List> requests = new ArrayList<>(); - requests.add(new Tuple<>(DeleteAction.NAME, - new DeleteRequest(SECURITY_INDEX_NAME, "type", "id"))); + requests.add(new Tuple<>(DeleteAction.NAME, new DeleteRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), "id"))); requests.add(new Tuple<>(BulkAction.NAME + "[s]", - createBulkShardRequest(SECURITY_INDEX_NAME, DeleteRequest::new))); - requests.add(new Tuple<>(UpdateAction.NAME, - new UpdateRequest(SECURITY_INDEX_NAME, "type", "id"))); - requests.add(new Tuple<>(IndexAction.NAME, - new IndexRequest(SECURITY_INDEX_NAME, "type", "id"))); + createBulkShardRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), DeleteRequest::new))); + requests.add(new Tuple<>(UpdateAction.NAME, new UpdateRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), "id"))); + requests.add(new Tuple<>(IndexAction.NAME, new IndexRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); requests.add(new Tuple<>(BulkAction.NAME + "[s]", - createBulkShardRequest(SECURITY_INDEX_NAME, IndexRequest::new))); - requests.add(new Tuple<>(SearchAction.NAME, new SearchRequest(SECURITY_INDEX_NAME))); + createBulkShardRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), IndexRequest::new))); + requests.add(new Tuple<>(SearchAction.NAME, new SearchRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); requests.add(new Tuple<>(TermVectorsAction.NAME, - new TermVectorsRequest(SECURITY_INDEX_NAME, "type", "id"))); - requests.add(new Tuple<>(GetAction.NAME, new GetRequest(SECURITY_INDEX_NAME, "type", "id"))); + new TermVectorsRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), "type", "id"))); + requests.add(new Tuple<>(GetAction.NAME, new GetRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), "type", "id"))); requests.add(new Tuple<>(TermVectorsAction.NAME, - new TermVectorsRequest(SECURITY_INDEX_NAME, "type", "id"))); - requests.add(new Tuple<>(IndicesAliasesAction.NAME, new IndicesAliasesRequest() - .addAliasAction(AliasActions.add().alias("security_alias").index(SECURITY_INDEX_NAME)))); - requests.add(new Tuple<>(ClusterHealthAction.NAME, new ClusterHealthRequest(SECURITY_INDEX_NAME))); + new TermVectorsRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), "type", "id"))); + requests.add(new Tuple<>(IndicesAliasesAction.NAME, + new IndicesAliasesRequest().addAliasAction(AliasActions.add().alias("security_alias").index(INTERNAL_SECURITY_INDEX)))); + requests.add( + new Tuple<>(ClusterHealthAction.NAME, new ClusterHealthRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); requests.add(new Tuple<>(ClusterHealthAction.NAME, - new ClusterHealthRequest(SECURITY_INDEX_NAME, "foo", "bar"))); + new ClusterHealthRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), "foo", "bar"))); for (final Tuple requestTuple : requests) { final String action = requestTuple.v1(); @@ -946,9 +970,12 @@ public void testSuperusersCanExecuteOperationAgainstSecurityIndexWithWildcard() ClusterState state = mock(ClusterState.class); when(clusterService.state()).thenReturn(state); when(state.metaData()).thenReturn(MetaData.builder() - .put(new IndexMetaData.Builder(SECURITY_INDEX_NAME) - .settings(Settings.builder().put("index.version.created", Version.CURRENT).build()) - .numberOfShards(1).numberOfReplicas(0).build(), true) + .put(new IndexMetaData.Builder(INTERNAL_SECURITY_INDEX) + .putAlias(new AliasMetaData.Builder(SECURITY_INDEX_NAME).build()) + .settings(Settings.builder().put("index.version.created", Version.CURRENT).build()) + .numberOfShards(1) + .numberOfReplicas(0) + .build(), true) .build()); final String requestId = AuditUtil.getOrGenerateRequestId(threadContext); @@ -956,7 +983,7 @@ public void testSuperusersCanExecuteOperationAgainstSecurityIndexWithWildcard() SearchRequest request = new SearchRequest("_all"); authorize(createAuthentication(superuser), action, request); verify(auditTrail).accessGranted(requestId, authentication, action, request, superuser.roles()); - assertThat(request.indices(), arrayContaining(".security")); + assertThat(request.indices(), arrayContainingInAnyOrder(INTERNAL_SECURITY_INDEX, SECURITY_INDEX_NAME)); } public void testAnonymousRolesAreAppliedToOtherUsers() { From 8e5f9c4b142f6ed0b5b0424c7de5d6c309989164 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 29 Jan 2019 11:18:30 -0800 Subject: [PATCH 025/100] Add OS/architecture classifier to distributions (#37881) This commit adds classifiers to the distributions indicating the OS (for archives) and platform. The current OSes are for windows, darwin (ie macos) and linux. This change will allow future OS/architecture specific changes to the distributions. Note the docs using distribution links have been updated, but will be reworked in a followup to make OS specific instructions for the archives. --- buildSrc/build.gradle | 8 ++- .../gradle/test/ClusterFormationTasks.groovy | 43 ++++++++++++--- .../gradle/vagrant/VagrantTestPlugin.groovy | 8 +-- distribution/archives/build.gradle | 24 +++++++-- .../{oss-tar => darwin-tar}/build.gradle | 0 .../{oss-zip => linux-tar}/build.gradle | 0 .../{tar => oss-darwin-tar}/build.gradle | 0 .../{zip => oss-linux-tar}/build.gradle | 0 .../archives/oss-windows-zip/build.gradle | 2 + .../archives/windows-zip/build.gradle | 2 + distribution/bwc/build.gradle | 33 ++++++++++-- distribution/docker/build.gradle | 4 +- distribution/packages/build.gradle | 4 +- docs/reference/setup/install/deb.asciidoc | 12 ++--- docs/reference/setup/install/rpm.asciidoc | 12 ++--- .../setup/install/zip-targz.asciidoc | 24 ++++----- .../setup/install/zip-windows.asciidoc | 4 +- .../packaging/test/ArchiveTestCase.java | 8 +-- .../packaging/test/DefaultTarTests.java | 2 +- .../test/DefaultWindowsServiceTests.java | 2 +- .../packaging/test/DefaultZipTests.java | 2 +- .../packaging/test/OssTarTests.java | 2 +- .../test/OssWindowsServiceTests.java | 2 +- .../packaging/test/OssZipTests.java | 2 +- .../test/RpmPreservationTestCase.java | 3 -- .../packaging/util/Archives.java | 2 +- .../packaging/util/Distribution.java | 52 ++++++++++++++----- .../packaging/util/Platforms.java | 1 + .../resources/packaging/utils/packages.bash | 10 +++- .../test/resources/packaging/utils/tar.bash | 2 +- settings.gradle | 10 ++-- 31 files changed, 196 insertions(+), 84 deletions(-) rename distribution/archives/{oss-tar => darwin-tar}/build.gradle (100%) rename distribution/archives/{oss-zip => linux-tar}/build.gradle (100%) rename distribution/archives/{tar => oss-darwin-tar}/build.gradle (100%) rename distribution/archives/{zip => oss-linux-tar}/build.gradle (100%) create mode 100644 distribution/archives/oss-windows-zip/build.gradle create mode 100644 distribution/archives/windows-zip/build.gradle diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 0acdc95c86bbf..2b5e4f2d24d1f 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -183,8 +183,12 @@ if (project != rootProject) { } dependencies { - distribution project(':distribution:archives:zip') - distribution project(':distribution:archives:oss-zip') + distribution project(':distribution:archives:windows-zip') + distribution project(':distribution:archives:oss-windows-zip') + distribution project(':distribution:archives:darwin-tar') + distribution project(':distribution:archives:oss-darwin-tar') + distribution project(':distribution:archives:linux-tar') + distribution project(':distribution:archives:oss-linux-tar') } String localDownloads = "${rootProject.buildDir}/local-downloads" diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index e38cb854a10b0..2fffd67215581 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -191,13 +191,21 @@ class ClusterFormationTasks { throw new GradleException("Unknown distribution: ${distro} in project ${project.path}") } Version version = Version.fromString(elasticsearchVersion) - String group = "downloads.zip" // dummy group, does not matter except for integ-test-zip, it is ignored by the fake ivy repo + String os = getOs() + String classifier = "${os}-x86_64" + String packaging = os.equals('windows') ? 'zip' : 'tar.gz' String artifactName = 'elasticsearch' if (distro.equals('oss') && Version.fromString(elasticsearchVersion).onOrAfter('6.3.0')) { artifactName += '-oss' } - String snapshotProject = distro == 'oss' ? 'oss-zip' : 'zip' Object dependency + String snapshotProject = "${os}-${os.equals('windows') ? 'zip' : 'tar'}" + if (version.before("7.0.0")) { + snapshotProject = "zip" + } + if (distro.equals("oss")) { + snapshotProject = "oss-" + snapshotProject + } boolean internalBuild = project.hasProperty('bwcVersions') VersionCollection.UnreleasedVersionInfo unreleasedInfo = null if (project.hasProperty('bwcVersions')) { @@ -205,11 +213,16 @@ class ClusterFormationTasks { unreleasedInfo = project.bwcVersions.unreleasedInfo(version) } if (unreleasedInfo != null) { - dependency = project.dependencies.project(path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: snapshotProject) + dependency = project.dependencies.project( + path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: snapshotProject) } else if (internalBuild && elasticsearchVersion.equals(VersionProperties.elasticsearch)) { dependency = project.dependencies.project(path: ":distribution:archives:${snapshotProject}") } else { - dependency = "${group}:${artifactName}:${elasticsearchVersion}@zip" + if (version.before('7.0.0')) { + classifier = "" // for bwc, before we had classifiers + } + // group does not matter as it is not used when we pull from the ivy repo that points to the download service + dependency = "dnm:${artifactName}:${elasticsearchVersion}${classifier}@${packaging}" } project.dependencies.add(configuration.name, dependency) } @@ -335,8 +348,15 @@ class ClusterFormationTasks { the elasticsearch source tree then this should be the version of elasticsearch built by the source tree. If it isn't then Bad Things(TM) will happen. */ Task extract = project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) { - from { - project.zipTree(configuration.singleFile) + if (getOs().equals("windows")) { + from { + project.zipTree(configuration.singleFile) + } + } else { + // macos and linux use tar + from { + project.tarTree(project.resources.gzip(configuration.singleFile)) + } } into node.baseDir } @@ -948,4 +968,15 @@ class ClusterFormationTasks { PluginPropertiesExtension extension = pluginProject.extensions.findByName('esplugin') return extension.name } + + /** Find the current OS */ + static String getOs() { + String os = "linux" + if (Os.FAMILY_WINDOWS) { + os = "windows" + } else if (Os.FAMILY_MAC) { + os = "darwin" + } + return os + } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy index 9f5364b78a896..8f54d63f4ca14 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy @@ -53,10 +53,10 @@ class VagrantTestPlugin implements Plugin { /** All distributions to bring into test VM, whether or not they are used **/ static final List DISTRIBUTIONS = unmodifiableList([ - 'archives:tar', - 'archives:oss-tar', - 'archives:zip', - 'archives:oss-zip', + 'archives:linux-tar', + 'archives:oss-linux-tar', + 'archives:windows-zip', + 'archives:oss-windows-zip', 'packages:rpm', 'packages:oss-rpm', 'packages:deb', diff --git a/distribution/archives/build.gradle b/distribution/archives/build.gradle index 7ab7e671ac214..3723e31b27f1e 100644 --- a/distribution/archives/build.gradle +++ b/distribution/archives/build.gradle @@ -105,13 +105,15 @@ task buildIntegTestZip(type: Zip) { with archiveFiles(transportModulesFiles, 'zip', true) } -task buildZip(type: Zip) { +task buildWindowsZip(type: Zip) { configure(commonZipConfig) + archiveClassifier = 'windows-x86_64' with archiveFiles(modulesFiles(false), 'zip', false) } -task buildOssZip(type: Zip) { +task buildOssWindowsZip(type: Zip) { configure(commonZipConfig) + archiveClassifier = 'windows-x86_64' with archiveFiles(modulesFiles(true), 'zip', true) } @@ -122,13 +124,27 @@ Closure commonTarConfig = { fileMode 0644 } -task buildTar(type: Tar) { +task buildDarwinTar(type: Tar) { configure(commonTarConfig) + archiveClassifier = 'darwin-x86_64' with archiveFiles(modulesFiles(false), 'tar', false) } -task buildOssTar(type: Tar) { +task buildOssDarwinTar(type: Tar) { configure(commonTarConfig) + archiveClassifier = 'darwin-x86_64' + with archiveFiles(modulesFiles(true), 'tar', true) +} + +task buildLinuxTar(type: Tar) { + configure(commonTarConfig) + archiveClassifier = 'linux-x86_64' + with archiveFiles(modulesFiles(false), 'tar', false) +} + +task buildOssLinuxTar(type: Tar) { + configure(commonTarConfig) + archiveClassifier = 'linux-x86_64' with archiveFiles(modulesFiles(true), 'tar', true) } diff --git a/distribution/archives/oss-tar/build.gradle b/distribution/archives/darwin-tar/build.gradle similarity index 100% rename from distribution/archives/oss-tar/build.gradle rename to distribution/archives/darwin-tar/build.gradle diff --git a/distribution/archives/oss-zip/build.gradle b/distribution/archives/linux-tar/build.gradle similarity index 100% rename from distribution/archives/oss-zip/build.gradle rename to distribution/archives/linux-tar/build.gradle diff --git a/distribution/archives/tar/build.gradle b/distribution/archives/oss-darwin-tar/build.gradle similarity index 100% rename from distribution/archives/tar/build.gradle rename to distribution/archives/oss-darwin-tar/build.gradle diff --git a/distribution/archives/zip/build.gradle b/distribution/archives/oss-linux-tar/build.gradle similarity index 100% rename from distribution/archives/zip/build.gradle rename to distribution/archives/oss-linux-tar/build.gradle diff --git a/distribution/archives/oss-windows-zip/build.gradle b/distribution/archives/oss-windows-zip/build.gradle new file mode 100644 index 0000000000000..4a6dde5fc0c92 --- /dev/null +++ b/distribution/archives/oss-windows-zip/build.gradle @@ -0,0 +1,2 @@ +// This file is intentionally blank. All configuration of the +// distribution is done in the parent project. diff --git a/distribution/archives/windows-zip/build.gradle b/distribution/archives/windows-zip/build.gradle new file mode 100644 index 0000000000000..4a6dde5fc0c92 --- /dev/null +++ b/distribution/archives/windows-zip/build.gradle @@ -0,0 +1,2 @@ +// This file is intentionally blank. All configuration of the +// distribution is done in the parent project. diff --git a/distribution/bwc/build.gradle b/distribution/bwc/build.gradle index 4557e0eb1dc4e..0ee597b449a41 100644 --- a/distribution/bwc/build.gradle +++ b/distribution/bwc/build.gradle @@ -122,17 +122,34 @@ bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unre Map artifactFiles = [:] List projectDirs = [] - List projects = ['zip', 'deb', 'rpm'] + List projects = ['deb', 'rpm'] + if (bwcVersion.onOrAfter('7.0.0')) { + projects.addAll(['windows-zip', 'darwin-tar', 'linux-tar']) + } else { + projects.add('zip') + } + for (String projectName : projects) { String baseDir = "distribution" + String classifier = "" + String extension = projectName + if (bwcVersion.onOrAfter('7.0.0') && (projectName.contains('zip') || projectName.contains('tar'))) { + int index = projectName.indexOf('-') + classifier = "-${projectName.substring(0, index)}-x86_64" + extension = projectName.substring(index + 1) + if (extension.equals('tar')) { + extension += '.gz' + } + } if (bwcVersion.onOrAfter('6.3.0')) { - baseDir += projectName == 'zip' ? '/archives' : '/packages' + baseDir += projectName.endsWith('zip') || projectName.endsWith('tar') ? '/archives' : '/packages' // add oss variant first projectDirs.add("${baseDir}/oss-${projectName}") - artifactFiles.put("oss-" + projectName, file("${checkoutDir}/${baseDir}/oss-${projectName}/build/distributions/elasticsearch-oss-${bwcVersion}-SNAPSHOT.${projectName}")) + artifactFiles.put("oss-" + projectName, file("${checkoutDir}/${baseDir}/oss-${projectName}/build/distributions/elasticsearch-oss-${bwcVersion}-SNAPSHOT${classifier}.${extension}")) } projectDirs.add("${baseDir}/${projectName}") - artifactFiles.put(projectName, file("${checkoutDir}/${baseDir}/${projectName}/build/distributions/elasticsearch-${bwcVersion}-SNAPSHOT.${projectName}")) + artifactFiles.put(projectName, + file("${checkoutDir}/${baseDir}/${projectName}/build/distributions/elasticsearch-${bwcVersion}-SNAPSHOT${classifier}.${extension}")) } Closure createRunBwcGradleTask = { name, extraConfig -> @@ -216,9 +233,15 @@ bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unre String artifactFileName = artifactFile.name String artifactName = artifactFileName.contains('oss') ? 'elasticsearch-oss' : 'elasticsearch' String suffix = artifactFile.toString()[-3..-1] + int archIndex = artifactFileName.indexOf('x86_64') + String classifier = '' + if (archIndex != -1) { + int osIndex = artifactFileName.lastIndexOf('-', archIndex - 2) + classifier = "${artifactFileName.substring(osIndex + 1, archIndex - 1)}-x86_64" + } configurations.create(projectName) artifacts { - it.add(projectName, [file: artifactFile, name: artifactName, type: suffix, builtBy: buildBwcVersion]) + it.add(projectName, [file: artifactFile, name: artifactName, classifier: classifier, type: suffix, builtBy: buildBwcVersion]) } } // make sure no dependencies were added to assemble; we want it to be a no-op diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 4d228e3c44fdc..8a5cea936bbcd 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -13,8 +13,8 @@ configurations { } dependencies { - dockerSource project(path: ":distribution:archives:tar") - ossDockerSource project(path: ":distribution:archives:oss-tar") + dockerSource project(path: ":distribution:archives:linux-tar") + ossDockerSource project(path: ":distribution:archives:oss-linux-tar") } ext.expansions = { oss -> diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle index 0b573ed9bad13..ad0f4a5cdec08 100644 --- a/distribution/packages/build.gradle +++ b/distribution/packages/build.gradle @@ -101,8 +101,9 @@ Closure commonPackageConfig(String type, boolean oss) { return { dependsOn "process${oss ? 'Oss' : ''}${type.capitalize()}Files" packageName "elasticsearch${oss ? '-oss' : ''}" + arch (type == 'deb' ? 'amd64' : 'X86_64') // Follow elasticsearch's file naming convention - archiveName "${packageName}-${project.version}.${type}" + archiveName "${packageName}-${project.version}-${archString}.${type}" String prefix = "${oss ? 'oss-' : ''}${type}" destinationDir = file("${prefix}/build/distributions") @@ -333,7 +334,6 @@ Closure commonRpmConfig(boolean oss) { packager 'Elasticsearch' version = project.version.replace('-', '_') release = '1' - arch 'NOARCH' os 'LINUX' distribution 'Elasticsearch' vendor 'Elasticsearch' diff --git a/docs/reference/setup/install/deb.asciidoc b/docs/reference/setup/install/deb.asciidoc index c5046f51ba112..97b4762338936 100644 --- a/docs/reference/setup/install/deb.asciidoc +++ b/docs/reference/setup/install/deb.asciidoc @@ -150,17 +150,17 @@ The Debian package for Elasticsearch v{version} can be downloaded from the websi ["source","sh",subs="attributes"] -------------------------------------------- -wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.deb -wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.deb.sha512 -shasum -a 512 -c elasticsearch-{version}.deb.sha512 <1> -sudo dpkg -i elasticsearch-{version}.deb +wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-amd64.deb +wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-amd64.deb.sha512 +shasum -a 512 -c elasticsearch-{version}-amd64.deb.sha512 <1> +sudo dpkg -i elasticsearch-{version}-amd64.deb -------------------------------------------- <1> Compares the SHA of the downloaded Debian package and the published checksum, which should output - `elasticsearch-{version}.deb: OK`. + `elasticsearch-{version}-amd64.deb: OK`. Alternatively, you can download the following package, which contains only features that are available under the Apache 2.0 license: -https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{version}.deb +https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{version}-amd64.deb endif::[] diff --git a/docs/reference/setup/install/rpm.asciidoc b/docs/reference/setup/install/rpm.asciidoc index 11add03573db1..a450e202b6896 100644 --- a/docs/reference/setup/install/rpm.asciidoc +++ b/docs/reference/setup/install/rpm.asciidoc @@ -135,17 +135,17 @@ The RPM for Elasticsearch v{version} can be downloaded from the website and inst ["source","sh",subs="attributes"] -------------------------------------------- -wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.rpm -wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.rpm.sha512 -shasum -a 512 -c elasticsearch-{version}.rpm.sha512 <1> -sudo rpm --install elasticsearch-{version}.rpm +wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-x86_64.rpm +wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-x86_64.rpm.sha512 +shasum -a 512 -c elasticsearch-{version}-x86_64.rpm.sha512 <1> +sudo rpm --install elasticsearch-{version}-x86_64.rpm -------------------------------------------- <1> Compares the SHA of the downloaded RPM and the published checksum, which should output - `elasticsearch-{version}.rpm: OK`. + `elasticsearch-{version}-x86_64.rpm: OK`. Alternatively, you can download the following package, which contains only features that are available under the Apache 2.0 license: -https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{version}.rpm +https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{version}-x86_64.rpm endif::[] diff --git a/docs/reference/setup/install/zip-targz.asciidoc b/docs/reference/setup/install/zip-targz.asciidoc index 735ca5b4ea0d1..d532438103754 100644 --- a/docs/reference/setup/install/zip-targz.asciidoc +++ b/docs/reference/setup/install/zip-targz.asciidoc @@ -32,19 +32,19 @@ The `.zip` archive for Elasticsearch v{version} can be downloaded and installed ["source","sh",subs="attributes"] -------------------------------------------- -wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.zip -wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.zip.sha512 -shasum -a 512 -c elasticsearch-{version}.zip.sha512 <1> -unzip elasticsearch-{version}.zip +wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-windows-x86_64.zip +wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-windows-x86_64.zip.sha512 +shasum -a 512 -c elasticsearch-{version}-windows-x86_64.zip.sha512 <1> +unzip elasticsearch-{version}-windows-x86_64.zip cd elasticsearch-{version}/ <2> -------------------------------------------- <1> Compares the SHA of the downloaded `.zip` archive and the published checksum, which should output - `elasticsearch-{version}.zip: OK`. + `elasticsearch-{version}-windows-x86_64.zip: OK`. <2> This directory is known as `$ES_HOME`. Alternatively, you can download the following package, which contains only features that are available under the Apache 2.0 license: -https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{version}.zip +https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{version}-windows-x86_64.zip endif::[] @@ -64,19 +64,19 @@ The `.tar.gz` archive for Elasticsearch v{version} can be downloaded and install ["source","sh",subs="attributes"] -------------------------------------------- -wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.tar.gz -wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.tar.gz.sha512 -shasum -a 512 -c elasticsearch-{version}.tar.gz.sha512 <1> -tar -xzf elasticsearch-{version}.tar.gz +wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-linux-x86_64.tar.gz +wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-linux-x86_64.tar.gz.sha512 +shasum -a 512 -c elasticsearch-{version}-linux-x86_64.tar.gz.sha512 <1> +tar -xzf elasticsearch-{version}-linux-x86_64.tar.gz cd elasticsearch-{version}/ <2> -------------------------------------------- <1> Compares the SHA of the downloaded `.tar.gz` archive and the published checksum, which should output - `elasticsearch-{version}.tar.gz: OK`. + `elasticsearch-{version}-linux-x86_64.tar.gz: OK`. <2> This directory is known as `$ES_HOME`. Alternatively, you can download the following package, which includes only Apache 2.0 licensed code: -https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{version}.tar.gz +https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{version}-linux-x86_64.tar.gz endif::[] diff --git a/docs/reference/setup/install/zip-windows.asciidoc b/docs/reference/setup/install/zip-windows.asciidoc index 254fb63f6157d..967b449bc972b 100644 --- a/docs/reference/setup/install/zip-windows.asciidoc +++ b/docs/reference/setup/install/zip-windows.asciidoc @@ -31,11 +31,11 @@ endif::[] ifeval::["{release-state}"!="unreleased"] -Download the `.zip` archive for Elasticsearch v{version} from: https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.zip +Download the `.zip` archive for Elasticsearch v{version} from: https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-windows-x86_64.zip Alternatively, you can download the following package, which contains only features that are available under the Apache 2.0 license: -https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{version}.zip +https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{version}-windows-x86_64.zip Unzip it with your favourite unzip tool. This will create a folder called +elasticsearch-{version}+, which we will refer to as `%ES_HOME%`. In a terminal diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java index 05b3603628adb..7f2299d1fc3ea 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java @@ -285,7 +285,7 @@ public void test90SecurityCliPackaging() { final Installation.Executables bin = installation.executables(); final Shell sh = new Shell(); - if (distribution().equals(Distribution.DEFAULT_TAR) || distribution().equals(Distribution.DEFAULT_ZIP)) { + if (distribution().equals(Distribution.DEFAULT_LINUX) || distribution().equals(Distribution.DEFAULT_WINDOWS)) { assertTrue(Files.exists(installation.lib.resolve("tools").resolve("security-cli"))); Platforms.onLinux(() -> { final Result result = sh.run(bin.elasticsearchCertutil + " help"); @@ -296,7 +296,7 @@ public void test90SecurityCliPackaging() { final Result result = sh.run(bin.elasticsearchCertutil + " help"); assertThat(result.stdout, containsString("Simplifies certificate creation for use with the Elastic Stack")); }); - } else if (distribution().equals(Distribution.OSS_TAR) || distribution().equals(Distribution.OSS_ZIP)) { + } else if (distribution().equals(Distribution.OSS_LINUX) || distribution().equals(Distribution.OSS_WINDOWS)) { assertFalse(Files.exists(installation.lib.resolve("tools").resolve("security-cli"))); } } @@ -312,7 +312,7 @@ public void test100ElasticsearchShardCliPackaging() { assertThat(result.stdout, containsString("A CLI tool to remove corrupted parts of unrecoverable shards")); }; - if (distribution().equals(Distribution.DEFAULT_TAR) || distribution().equals(Distribution.DEFAULT_ZIP)) { + if (distribution().equals(Distribution.DEFAULT_LINUX) || distribution().equals(Distribution.DEFAULT_WINDOWS)) { Platforms.onLinux(action); Platforms.onWindows(action); } @@ -330,7 +330,7 @@ public void test110ElasticsearchNodeCliPackaging() { containsString("A CLI tool to unsafely recover a cluster after the permanent loss of too many master-eligible nodes")); }; - if (distribution().equals(Distribution.DEFAULT_TAR) || distribution().equals(Distribution.DEFAULT_ZIP)) { + if (distribution().equals(Distribution.DEFAULT_LINUX) || distribution().equals(Distribution.DEFAULT_WINDOWS)) { Platforms.onLinux(action); Platforms.onWindows(action); } diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultTarTests.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultTarTests.java index 9b359a329c1bb..b6c633c0e72a2 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultTarTests.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultTarTests.java @@ -25,6 +25,6 @@ public class DefaultTarTests extends ArchiveTestCase { @Override protected Distribution distribution() { - return Distribution.DEFAULT_TAR; + return Distribution.DEFAULT_LINUX; } } diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultWindowsServiceTests.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultWindowsServiceTests.java index 072c3da68868f..6fedd3b89db70 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultWindowsServiceTests.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultWindowsServiceTests.java @@ -25,6 +25,6 @@ public class DefaultWindowsServiceTests extends WindowsServiceTestCase { @Override protected Distribution distribution() { - return Distribution.DEFAULT_ZIP; + return Distribution.DEFAULT_WINDOWS; } } diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultZipTests.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultZipTests.java index d9a6353a8c6f3..852535188cf9b 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultZipTests.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultZipTests.java @@ -25,6 +25,6 @@ public class DefaultZipTests extends ArchiveTestCase { @Override protected Distribution distribution() { - return Distribution.DEFAULT_ZIP; + return Distribution.DEFAULT_WINDOWS; } } diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssTarTests.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssTarTests.java index 86637fc9d48e3..3e72f1da5cbb0 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssTarTests.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssTarTests.java @@ -25,6 +25,6 @@ public class OssTarTests extends ArchiveTestCase { @Override protected Distribution distribution() { - return Distribution.OSS_TAR; + return Distribution.OSS_LINUX; } } diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssWindowsServiceTests.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssWindowsServiceTests.java index f4de95f67b6db..bfa220c6aaf22 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssWindowsServiceTests.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssWindowsServiceTests.java @@ -25,6 +25,6 @@ public class OssWindowsServiceTests extends WindowsServiceTestCase { @Override protected Distribution distribution() { - return Distribution.OSS_ZIP; + return Distribution.OSS_WINDOWS; } } diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssZipTests.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssZipTests.java index b6cd1e596a09b..418ba6d344650 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssZipTests.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssZipTests.java @@ -25,6 +25,6 @@ public class OssZipTests extends ArchiveTestCase { @Override protected Distribution distribution() { - return Distribution.OSS_ZIP; + return Distribution.OSS_WINDOWS; } } diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/RpmPreservationTestCase.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/RpmPreservationTestCase.java index 4787766ae3b61..36558ea2429cc 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/RpmPreservationTestCase.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/RpmPreservationTestCase.java @@ -43,10 +43,7 @@ import static org.elasticsearch.packaging.util.Platforms.isRPM; import static org.elasticsearch.packaging.util.Platforms.isSystemd; import static org.hamcrest.CoreMatchers.notNullValue; -import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeThat; import static org.junit.Assume.assumeTrue; diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java index db48ed753b271..ed579c35baf56 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java @@ -70,7 +70,7 @@ public static Installation installArchive(Distribution distribution, Path fullIn final Path baseInstallPath = fullInstallPath.getParent(); final Path extractedPath = baseInstallPath.resolve("elasticsearch-" + version); - assertThat("distribution file must exist", Files.exists(distributionFile), is(true)); + assertThat("distribution file must exist: " + distributionFile.toString(), Files.exists(distributionFile), is(true)); assertThat("elasticsearch must not already be installed", lsGlob(baseInstallPath, "elasticsearch*"), empty()); if (distribution.packaging == Distribution.Packaging.TAR) { diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Distribution.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Distribution.java index 7e1067fdb351c..5ed1bcb5a3d1f 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Distribution.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Distribution.java @@ -19,28 +19,46 @@ package org.elasticsearch.packaging.util; +import java.util.Locale; + public enum Distribution { - OSS_TAR(Packaging.TAR, Flavor.OSS), - OSS_ZIP(Packaging.ZIP, Flavor.OSS), - OSS_DEB(Packaging.DEB, Flavor.OSS), - OSS_RPM(Packaging.RPM, Flavor.OSS), + OSS_LINUX(Packaging.TAR, Platform.LINUX, Flavor.OSS), + OSS_WINDOWS(Packaging.ZIP, Platform.WINDOWS, Flavor.OSS), + OSS_DARWIN(Packaging.TAR, Platform.DARWIN, Flavor.OSS), + OSS_DEB(Packaging.DEB, Platform.LINUX, Flavor.OSS), + OSS_RPM(Packaging.RPM, Platform.LINUX, Flavor.OSS), - DEFAULT_TAR(Packaging.TAR, Flavor.DEFAULT), - DEFAULT_ZIP(Packaging.ZIP, Flavor.DEFAULT), - DEFAULT_DEB(Packaging.DEB, Flavor.DEFAULT), - DEFAULT_RPM(Packaging.RPM, Flavor.DEFAULT); + DEFAULT_LINUX(Packaging.TAR, Platform.LINUX, Flavor.DEFAULT), + DEFAULT_WINDOWS(Packaging.ZIP, Platform.WINDOWS, Flavor.DEFAULT), + DEFAULT_DARWIN(Packaging.TAR, Platform.DARWIN, Flavor.DEFAULT), + DEFAULT_DEB(Packaging.DEB, Platform.LINUX, Flavor.DEFAULT), + DEFAULT_RPM(Packaging.RPM, Platform.LINUX, Flavor.DEFAULT); public final Packaging packaging; + public final Platform platform; public final Flavor flavor; - Distribution(Packaging packaging, Flavor flavor) { + Distribution(Packaging packaging, Platform platform, Flavor flavor) { this.packaging = packaging; + this.platform = platform; this.flavor = flavor; } public String filename(String version) { - return flavor.name + "-" + version + packaging.extension; + String architecture = ""; + if (version.startsWith("6.") == false) { + + if (packaging == Packaging.DEB) { + architecture = "-amd64"; + } else { + if (packaging != Packaging.RPM) { + architecture = "-" + platform.toString(); + } + architecture += "-x86_64"; + } + } + return flavor.name + "-" + version + architecture + packaging.extension; } public boolean isDefault() { @@ -53,8 +71,8 @@ public boolean isOSS() { public enum Packaging { - TAR(".tar.gz", Platforms.LINUX), - ZIP(".zip", true), + TAR(".tar.gz", Platforms.LINUX || Platforms.DARWIN), + ZIP(".zip", Platforms.WINDOWS), DEB(".deb", Platforms.isDPKG()), RPM(".rpm", Platforms.isRPM()); @@ -70,6 +88,16 @@ public enum Packaging { } } + public enum Platform { + LINUX, + WINDOWS, + DARWIN; + + public String toString() { + return name().toLowerCase(Locale.ROOT); + } + } + public enum Flavor { OSS("elasticsearch-oss"), diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Platforms.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Platforms.java index c7ca1284ca69a..dbac9c88d26c9 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Platforms.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Platforms.java @@ -27,6 +27,7 @@ public class Platforms { public static final String OS_NAME = System.getProperty("os.name"); public static final boolean LINUX = OS_NAME.startsWith("Linux"); public static final boolean WINDOWS = OS_NAME.startsWith("Windows"); + public static final boolean DARWIN = OS_NAME.startsWith("Mac OS X"); public static String getOsRelease() { if (LINUX) { diff --git a/qa/vagrant/src/test/resources/packaging/utils/packages.bash b/qa/vagrant/src/test/resources/packaging/utils/packages.bash index d86f1c64e2e69..0285e3682b3f7 100644 --- a/qa/vagrant/src/test/resources/packaging/utils/packages.bash +++ b/qa/vagrant/src/test/resources/packaging/utils/packages.bash @@ -73,10 +73,16 @@ install_package() { ;; esac done + local rpm_classifier="-x86_64" + local deb_classifier="-amd64" + if [[ $version == 6* ]]; then + rpm_classifier="" + deb_classifier="" + fi if is_rpm; then - rpm $rpmCommand $PACKAGE_NAME-$version.rpm + rpm $rpmCommand $PACKAGE_NAME-$version$rpm_classifier.rpm elif is_dpkg; then - run dpkg $dpkgCommand -i $PACKAGE_NAME-$version.deb + run dpkg $dpkgCommand -i $PACKAGE_NAME-$version$deb_classifier.deb [[ "$status" -eq 0 ]] || { echo "dpkg failed:" echo "$output" diff --git a/qa/vagrant/src/test/resources/packaging/utils/tar.bash b/qa/vagrant/src/test/resources/packaging/utils/tar.bash index eb2e39274e29e..142215a703682 100644 --- a/qa/vagrant/src/test/resources/packaging/utils/tar.bash +++ b/qa/vagrant/src/test/resources/packaging/utils/tar.bash @@ -40,7 +40,7 @@ install_archive() { echo "Unpacking tarball to $ESHOME" rm -rf /tmp/untar mkdir -p /tmp/untar - tar -xzpf "${PACKAGE_NAME}-${version}.tar.gz" -C /tmp/untar + tar -xzpf "${PACKAGE_NAME}-${version}-linux-x86_64.tar.gz" -C /tmp/untar find /tmp/untar -depth -type d -name 'elasticsearch*' -exec mv {} "$ESHOME" \; > /dev/null diff --git a/settings.gradle b/settings.gradle index d5eebd6f66fe2..18f5f63332e00 100644 --- a/settings.gradle +++ b/settings.gradle @@ -16,10 +16,12 @@ List projects = [ 'client:benchmark', 'benchmarks', 'distribution:archives:integ-test-zip', - 'distribution:archives:oss-zip', - 'distribution:archives:zip', - 'distribution:archives:oss-tar', - 'distribution:archives:tar', + 'distribution:archives:oss-windows-zip', + 'distribution:archives:windows-zip', + 'distribution:archives:oss-darwin-tar', + 'distribution:archives:darwin-tar', + 'distribution:archives:oss-linux-tar', + 'distribution:archives:linux-tar', 'distribution:docker', 'distribution:packages:oss-deb', 'distribution:packages:deb', From e9332331a39efdb42be204c7df0f04d4088d0d6e Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Tue, 29 Jan 2019 21:20:09 +0200 Subject: [PATCH 026/100] SQL: Make error msg for validation of 2nd arg of PERCENTILE[_RANK] consistent (#37937) Use `first` and `second` instead of `1st` and `2nd`. --- .../xpack/sql/expression/function/aggregate/Percentile.java | 2 +- .../sql/expression/function/aggregate/PercentileRank.java | 2 +- .../sql/analysis/analyzer/VerifierErrorMessagesTests.java | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Percentile.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Percentile.java index 295932cd99c5e..76c7bda320012 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Percentile.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Percentile.java @@ -43,7 +43,7 @@ public Percentile replaceChildren(List newChildren) { @Override protected TypeResolution resolveType() { if (!percent.foldable()) { - return new TypeResolution(format(null, "2nd argument of PERCENTILE must be a constant, received [{}]", + return new TypeResolution(format(null, "Second argument of PERCENTILE must be a constant, received [{}]", Expressions.name(percent))); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/PercentileRank.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/PercentileRank.java index 92bc794b248da..b30b38a01b6c5 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/PercentileRank.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/PercentileRank.java @@ -43,7 +43,7 @@ public Expression replaceChildren(List newChildren) { @Override protected TypeResolution resolveType() { if (!value.foldable()) { - return new TypeResolution(format(null, "2nd argument of PERCENTILE_RANK must be a constant, received [{}]", + return new TypeResolution(format(null, "Second argument of PERCENTILE_RANK must be a constant, received [{}]", Expressions.name(value))); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java index 3316b179f50cb..4279910e0e03b 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java @@ -538,12 +538,12 @@ public void testAggsInHistogram() { } public void testErrorMessageForPercentileWithSecondArgBasedOnAField() { - assertEquals("1:8: 2nd argument of PERCENTILE must be a constant, received [ABS(int)]", + assertEquals("1:8: Second argument of PERCENTILE must be a constant, received [ABS(int)]", error("SELECT PERCENTILE(int, ABS(int)) FROM test")); } public void testErrorMessageForPercentileRankWithSecondArgBasedOnAField() { - assertEquals("1:8: 2nd argument of PERCENTILE_RANK must be a constant, received [ABS(int)]", + assertEquals("1:8: Second argument of PERCENTILE_RANK must be a constant, received [ABS(int)]", error("SELECT PERCENTILE_RANK(int, ABS(int)) FROM test")); } } From f3f9cabd67681009f5eadee369b1dc1e24d8c67d Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 29 Jan 2019 12:29:06 -0700 Subject: [PATCH 027/100] Add timeout for ccr recovery action (#37840) This is related to #35975. It adds a action timeout setting that allows timeouts to be applied to the individual transport actions that are used during a ccr recovery. --- .../test/transport/MockTransportService.java | 10 +++ .../transport/StubbableConnectionManager.java | 2 + .../test/transport/StubbableTransport.java | 8 ++ .../elasticsearch/xpack/ccr/CcrSettings.java | 19 +++++ .../xpack/ccr/repository/CcrRepository.java | 41 ++++++---- .../elasticsearch/xpack/CcrIntegTestCase.java | 5 +- .../xpack/ccr/CcrRepositoryIT.java | 77 +++++++++++++++++++ .../CcrRestoreSourceServiceTests.java | 2 +- 8 files changed, 145 insertions(+), 19 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java index 1ff1fa37fc120..469a4cfb4d55e 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java @@ -406,6 +406,16 @@ public boolean addSendBehavior(TransportAddress transportAddress, StubbableTrans return transport().addSendBehavior(transportAddress, sendBehavior); } + /** + * Adds a send behavior that is the default send behavior. + * + * @return {@code true} if no default send behavior was registered + */ + public boolean addSendBehavior(StubbableTransport.SendRequestBehavior behavior) { + return transport().setDefaultSendBehavior(behavior); + } + + /** * Adds a new connect behavior that is used for creating connections with the given delegate service. * diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableConnectionManager.java b/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableConnectionManager.java index 994037be07a92..41ac87f0af576 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableConnectionManager.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableConnectionManager.java @@ -69,7 +69,9 @@ public boolean setDefaultNodeConnectedBehavior(NodeConnectedBehavior behavior) { } public void clearBehaviors() { + defaultGetConnectionBehavior = ConnectionManager::getConnection; getConnectionBehaviors.clear(); + defaultNodeConnectedBehavior = ConnectionManager::nodeConnected; nodeConnectedBehaviors.clear(); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java index 4f0df85fd50c8..673ed49387570 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java @@ -55,6 +55,12 @@ public StubbableTransport(Transport transport) { this.delegate = transport; } + boolean setDefaultSendBehavior(SendRequestBehavior sendBehavior) { + SendRequestBehavior prior = defaultSendRequest; + defaultSendRequest = sendBehavior; + return prior == null; + } + public boolean setDefaultConnectBehavior(OpenConnectionBehavior openConnectionBehavior) { OpenConnectionBehavior prior = this.defaultConnectBehavior; this.defaultConnectBehavior = openConnectionBehavior; @@ -70,7 +76,9 @@ boolean addConnectBehavior(TransportAddress transportAddress, OpenConnectionBeha } void clearBehaviors() { + this.defaultSendRequest = null; sendBehaviors.clear(); + this.defaultConnectBehavior = null; connectBehaviors.clear(); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrSettings.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrSettings.java index d6262a7711dad..625429dc0abc6 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrSettings.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrSettings.java @@ -57,6 +57,13 @@ public final class CcrSettings { Setting.timeSetting("ccr.indices.recovery.recovery_activity_timeout", TimeValue.timeValueSeconds(60), Setting.Property.Dynamic, Setting.Property.NodeScope); + /** + * The timeout value to use for requests made as part of ccr recovery process. + * */ + public static final Setting INDICES_RECOVERY_ACTION_TIMEOUT_SETTING = + Setting.positiveTimeSetting("ccr.indices.recovery.internal_action_timeout", TimeValue.timeValueSeconds(60), + Property.Dynamic, Property.NodeScope); + /** * The settings defined by CCR. * @@ -67,6 +74,7 @@ static List> getSettings() { XPackSettings.CCR_ENABLED_SETTING, CCR_FOLLOWING_INDEX_SETTING, RECOVERY_MAX_BYTES_PER_SECOND, + INDICES_RECOVERY_ACTION_TIMEOUT_SETTING, INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING, CCR_AUTO_FOLLOW_WAIT_FOR_METADATA_TIMEOUT, CCR_WAIT_FOR_METADATA_TIMEOUT); @@ -74,12 +82,15 @@ static List> getSettings() { private final CombinedRateLimiter ccrRateLimiter; private volatile TimeValue recoveryActivityTimeout; + private volatile TimeValue recoveryActionTimeout; public CcrSettings(Settings settings, ClusterSettings clusterSettings) { this.recoveryActivityTimeout = INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING.get(settings); + this.recoveryActionTimeout = INDICES_RECOVERY_ACTION_TIMEOUT_SETTING.get(settings); this.ccrRateLimiter = new CombinedRateLimiter(RECOVERY_MAX_BYTES_PER_SECOND.get(settings)); clusterSettings.addSettingsUpdateConsumer(RECOVERY_MAX_BYTES_PER_SECOND, this::setMaxBytesPerSec); clusterSettings.addSettingsUpdateConsumer(INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING, this::setRecoveryActivityTimeout); + clusterSettings.addSettingsUpdateConsumer(INDICES_RECOVERY_ACTION_TIMEOUT_SETTING, this::setRecoveryActionTimeout); } private void setMaxBytesPerSec(ByteSizeValue maxBytesPerSec) { @@ -90,6 +101,10 @@ private void setRecoveryActivityTimeout(TimeValue recoveryActivityTimeout) { this.recoveryActivityTimeout = recoveryActivityTimeout; } + private void setRecoveryActionTimeout(TimeValue recoveryActionTimeout) { + this.recoveryActionTimeout = recoveryActionTimeout; + } + public CombinedRateLimiter getRateLimiter() { return ccrRateLimiter; } @@ -97,4 +112,8 @@ public CombinedRateLimiter getRateLimiter() { public TimeValue getRecoveryActivityTimeout() { return recoveryActivityTimeout; } + + public TimeValue getRecoveryActionTimeout() { + return recoveryActionTimeout; + } } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java index 7ca95a14909ae..bcf0e5f6dc6e9 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java @@ -127,7 +127,8 @@ public RepositoryMetaData getMetadata() { public SnapshotInfo getSnapshotInfo(SnapshotId snapshotId) { assert SNAPSHOT_ID.equals(snapshotId) : "RemoteClusterRepository only supports " + SNAPSHOT_ID + " as the SnapshotId"; Client remoteClient = client.getRemoteClusterClient(remoteClusterAlias); - ClusterStateResponse response = remoteClient.admin().cluster().prepareState().clear().setMetaData(true).setNodes(true).get(); + ClusterStateResponse response = remoteClient.admin().cluster().prepareState().clear().setMetaData(true).setNodes(true) + .get(ccrSettings.getRecoveryActionTimeout()); ImmutableOpenMap indicesMap = response.getState().metaData().indices(); ArrayList indices = new ArrayList<>(indicesMap.size()); indicesMap.keysIt().forEachRemaining(indices::add); @@ -141,7 +142,8 @@ public MetaData getSnapshotGlobalMetaData(SnapshotId snapshotId) { Client remoteClient = client.getRemoteClusterClient(remoteClusterAlias); // We set a single dummy index name to avoid fetching all the index data ClusterStateRequest clusterStateRequest = CcrRequests.metaDataRequest("dummy_index_name"); - ClusterStateResponse clusterState = remoteClient.admin().cluster().state(clusterStateRequest).actionGet(); + ClusterStateResponse clusterState = remoteClient.admin().cluster().state(clusterStateRequest) + .actionGet(ccrSettings.getRecoveryActionTimeout()); return clusterState.getState().metaData(); } @@ -152,13 +154,14 @@ public IndexMetaData getSnapshotIndexMetaData(SnapshotId snapshotId, IndexId ind Client remoteClient = client.getRemoteClusterClient(remoteClusterAlias); ClusterStateRequest clusterStateRequest = CcrRequests.metaDataRequest(leaderIndex); - ClusterStateResponse clusterState = remoteClient.admin().cluster().state(clusterStateRequest).actionGet(); + ClusterStateResponse clusterState = remoteClient.admin().cluster().state(clusterStateRequest) + .actionGet(ccrSettings.getRecoveryActionTimeout()); // Validates whether the leader cluster has been configured properly: PlainActionFuture future = PlainActionFuture.newFuture(); IndexMetaData leaderIndexMetaData = clusterState.getState().metaData().index(leaderIndex); ccrLicenseChecker.fetchLeaderHistoryUUIDs(remoteClient, leaderIndexMetaData, future::onFailure, future::onResponse); - String[] leaderHistoryUUIDs = future.actionGet(); + String[] leaderHistoryUUIDs = future.actionGet(ccrSettings.getRecoveryActionTimeout()); IndexMetaData.Builder imdBuilder = IndexMetaData.builder(leaderIndex); // Adding the leader index uuid for each shard as custom metadata: @@ -188,7 +191,8 @@ public IndexMetaData getSnapshotIndexMetaData(SnapshotId snapshotId, IndexId ind @Override public RepositoryData getRepositoryData() { Client remoteClient = client.getRemoteClusterClient(remoteClusterAlias); - ClusterStateResponse response = remoteClient.admin().cluster().prepareState().clear().setMetaData(true).get(); + ClusterStateResponse response = remoteClient.admin().cluster().prepareState().clear().setMetaData(true) + .get(ccrSettings.getRecoveryActionTimeout()); MetaData remoteMetaData = response.getState().getMetaData(); Map copiedSnapshotIds = new HashMap<>(); @@ -298,7 +302,8 @@ public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, Ve private void maybeUpdateMappings(Client localClient, Client remoteClient, Index leaderIndex, IndexSettings followerIndexSettings) { ClusterStateRequest clusterStateRequest = CcrRequests.metaDataRequest(leaderIndex.getName()); - ClusterStateResponse clusterState = remoteClient.admin().cluster().state(clusterStateRequest).actionGet(); + ClusterStateResponse clusterState = remoteClient.admin().cluster().state(clusterStateRequest) + .actionGet(ccrSettings.getRecoveryActionTimeout()); IndexMetaData leaderIndexMetadata = clusterState.getState().metaData().getIndexSafe(leaderIndex); long leaderMappingVersion = leaderIndexMetadata.getMappingVersion(); @@ -306,7 +311,7 @@ private void maybeUpdateMappings(Client localClient, Client remoteClient, Index Index followerIndex = followerIndexSettings.getIndex(); MappingMetaData mappingMetaData = leaderIndexMetadata.mapping(); PutMappingRequest putMappingRequest = CcrRequests.putMappingRequest(followerIndex.getName(), mappingMetaData); - localClient.admin().indices().putMapping(putMappingRequest).actionGet(); + localClient.admin().indices().putMapping(putMappingRequest).actionGet(ccrSettings.getRecoveryActionTimeout()); } } @@ -314,9 +319,9 @@ private RestoreSession openSession(String repositoryName, Client remoteClient, S RecoveryState recoveryState) { String sessionUUID = UUIDs.randomBase64UUID(); PutCcrRestoreSessionAction.PutCcrRestoreSessionResponse response = remoteClient.execute(PutCcrRestoreSessionAction.INSTANCE, - new PutCcrRestoreSessionRequest(sessionUUID, leaderShardId)).actionGet(); + new PutCcrRestoreSessionRequest(sessionUUID, leaderShardId)).actionGet(ccrSettings.getRecoveryActionTimeout()); return new RestoreSession(repositoryName, remoteClient, sessionUUID, response.getNode(), indexShard, recoveryState, - response.getStoreFileMetaData(), ccrSettings.getRateLimiter(), throttledTime::inc); + response.getStoreFileMetaData(), ccrSettings, throttledTime::inc); } private static class RestoreSession extends FileRestoreContext implements Closeable { @@ -327,18 +332,18 @@ private static class RestoreSession extends FileRestoreContext implements Closea private final String sessionUUID; private final DiscoveryNode node; private final Store.MetadataSnapshot sourceMetaData; - private final CombinedRateLimiter rateLimiter; + private final CcrSettings ccrSettings; private final LongConsumer throttleListener; RestoreSession(String repositoryName, Client remoteClient, String sessionUUID, DiscoveryNode node, IndexShard indexShard, - RecoveryState recoveryState, Store.MetadataSnapshot sourceMetaData, CombinedRateLimiter rateLimiter, + RecoveryState recoveryState, Store.MetadataSnapshot sourceMetaData, CcrSettings ccrSettings, LongConsumer throttleListener) { super(repositoryName, indexShard, SNAPSHOT_ID, recoveryState, BUFFER_SIZE); this.remoteClient = remoteClient; this.sessionUUID = sessionUUID; this.node = node; this.sourceMetaData = sourceMetaData; - this.rateLimiter = rateLimiter; + this.ccrSettings = ccrSettings; this.throttleListener = throttleListener; } @@ -354,14 +359,14 @@ void restoreFiles() throws IOException { @Override protected InputStream fileInputStream(BlobStoreIndexShardSnapshot.FileInfo fileInfo) { - return new RestoreFileInputStream(remoteClient, sessionUUID, node, fileInfo.metadata(), rateLimiter, throttleListener); + return new RestoreFileInputStream(remoteClient, sessionUUID, node, fileInfo.metadata(), ccrSettings, throttleListener); } @Override public void close() { ClearCcrRestoreSessionRequest clearRequest = new ClearCcrRestoreSessionRequest(sessionUUID, node); ClearCcrRestoreSessionAction.ClearCcrRestoreSessionResponse response = - remoteClient.execute(ClearCcrRestoreSessionAction.INSTANCE, clearRequest).actionGet(); + remoteClient.execute(ClearCcrRestoreSessionAction.INSTANCE, clearRequest).actionGet(ccrSettings.getRecoveryActionTimeout()); } } @@ -372,17 +377,19 @@ private static class RestoreFileInputStream extends InputStream { private final DiscoveryNode node; private final StoreFileMetaData fileToRecover; private final CombinedRateLimiter rateLimiter; + private final CcrSettings ccrSettings; private final LongConsumer throttleListener; private long pos = 0; private RestoreFileInputStream(Client remoteClient, String sessionUUID, DiscoveryNode node, StoreFileMetaData fileToRecover, - CombinedRateLimiter rateLimiter, LongConsumer throttleListener) { + CcrSettings ccrSettings, LongConsumer throttleListener) { this.remoteClient = remoteClient; this.sessionUUID = sessionUUID; this.node = node; this.fileToRecover = fileToRecover; - this.rateLimiter = rateLimiter; + this.ccrSettings = ccrSettings; + this.rateLimiter = ccrSettings.getRateLimiter(); this.throttleListener = throttleListener; } @@ -407,7 +414,7 @@ public int read(byte[] bytes, int off, int len) throws IOException { String fileName = fileToRecover.name(); GetCcrRestoreFileChunkRequest request = new GetCcrRestoreFileChunkRequest(node, sessionUUID, fileName, bytesRequested); GetCcrRestoreFileChunkAction.GetCcrRestoreFileChunkResponse response = - remoteClient.execute(GetCcrRestoreFileChunkAction.INSTANCE, request).actionGet(); + remoteClient.execute(GetCcrRestoreFileChunkAction.INSTANCE, request).actionGet(ccrSettings.getRecoveryActionTimeout()); BytesReference fileChunk = response.getChunk(); int bytesReceived = fileChunk.length(); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java index 4a6c5411737c6..c4fdeb116ae86 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java @@ -65,7 +65,9 @@ import org.elasticsearch.test.NodeConfigurationSource; import org.elasticsearch.test.TestCluster; import org.elasticsearch.test.discovery.TestZenDiscovery; +import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.transport.nio.MockNioTransportPlugin; import org.elasticsearch.xpack.ccr.CcrSettings; import org.elasticsearch.xpack.ccr.LocalStateCcr; import org.elasticsearch.xpack.ccr.index.engine.FollowingEngine; @@ -120,7 +122,8 @@ public final void startClusters() throws Exception { stopClusters(); Collection> mockPlugins = Arrays.asList(ESIntegTestCase.TestSeedPlugin.class, - TestZenDiscovery.TestPlugin.class, MockHttpTransport.TestPlugin.class, getTestTransportPlugin()); + TestZenDiscovery.TestPlugin.class, MockHttpTransport.TestPlugin.class, MockTransportService.TestPlugin.class, + MockNioTransportPlugin.class); InternalTestCluster leaderCluster = new InternalTestCluster(randomLong(), createTempDir(), true, true, numberOfNodesPerCluster(), numberOfNodesPerCluster(), UUIDs.randomBase64UUID(random()), createNodeConfigurationSource(null), 0, "leader", mockPlugins, diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java index ee02241978cfe..f22857939e0d1 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java @@ -6,6 +6,8 @@ package org.elasticsearch.xpack.ccr; +import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; @@ -34,8 +36,10 @@ import org.elasticsearch.snapshots.RestoreInfo; import org.elasticsearch.snapshots.RestoreService; import org.elasticsearch.snapshots.Snapshot; +import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.CcrIntegTestCase; +import org.elasticsearch.xpack.ccr.action.repositories.GetCcrRestoreFileChunkAction; import org.elasticsearch.xpack.ccr.repository.CcrRepository; import org.elasticsearch.xpack.ccr.repository.CcrRestoreSourceService; @@ -51,6 +55,7 @@ import static org.elasticsearch.snapshots.RestoreService.restoreInProgress; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; // TODO: Fold this integration test into a more expansive integration test as more bootstrap from remote work // TODO: is completed. @@ -308,6 +313,78 @@ public void testRateLimitingIsEmployed() throws Exception { } } + public void testIndividualActionsTimeout() throws Exception { + ClusterUpdateSettingsRequest settingsRequest = new ClusterUpdateSettingsRequest(); + TimeValue timeValue = TimeValue.timeValueMillis(100); + settingsRequest.persistentSettings(Settings.builder().put(CcrSettings.INDICES_RECOVERY_ACTION_TIMEOUT_SETTING.getKey(), timeValue)); + assertAcked(followerClient().admin().cluster().updateSettings(settingsRequest).actionGet()); + + String leaderClusterRepoName = CcrRepository.NAME_PREFIX + "leader_cluster"; + String leaderIndex = "index1"; + String followerIndex = "index2"; + + final int numberOfPrimaryShards = randomIntBetween(1, 3); + final String leaderIndexSettings = getIndexSettings(numberOfPrimaryShards, between(0, 1), + singletonMap(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true")); + assertAcked(leaderClient().admin().indices().prepareCreate(leaderIndex).setSource(leaderIndexSettings, XContentType.JSON)); + ensureLeaderGreen(leaderIndex); + + List transportServices = new ArrayList<>(); + + for (TransportService transportService : getFollowerCluster().getDataOrMasterNodeInstances(TransportService.class)) { + MockTransportService mockTransportService = (MockTransportService) transportService; + transportServices.add(mockTransportService); + mockTransportService.addSendBehavior((connection, requestId, action, request, options) -> { + if (action.equals(GetCcrRestoreFileChunkAction.NAME) == false) { + connection.sendRequest(requestId, action, request, options); + } + }); + } + + logger.info("--> indexing some data"); + for (int i = 0; i < 100; i++) { + final String source = String.format(Locale.ROOT, "{\"f\":%d}", i); + leaderClient().prepareIndex("index1", "doc", Integer.toString(i)).setSource(source, XContentType.JSON).get(); + } + + leaderClient().admin().indices().prepareFlush(leaderIndex).setForce(true).setWaitIfOngoing(true).get(); + + Settings.Builder settingsBuilder = Settings.builder() + .put(IndexMetaData.SETTING_INDEX_PROVIDED_NAME, followerIndex) + .put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true); + RestoreSnapshotRequest restoreRequest = new RestoreSnapshotRequest(leaderClusterRepoName, CcrRepository.LATEST) + .indices(leaderIndex).indicesOptions(indicesOptions).renamePattern("^(.*)$") + .renameReplacement(followerIndex).masterNodeTimeout(new TimeValue(1L, TimeUnit.HOURS)) + .indexSettings(settingsBuilder); + + final RestoreService restoreService = getFollowerCluster().getCurrentMasterNodeInstance(RestoreService.class); + final ClusterService clusterService = getFollowerCluster().getCurrentMasterNodeInstance(ClusterService.class); + PlainActionFuture future = PlainActionFuture.newFuture(); + restoreService.restoreSnapshot(restoreRequest, waitForRestore(clusterService, future)); + + // Depending on when the timeout occurs this can fail in two ways. If it times-out when fetching + // metadata this will throw an exception. If it times-out when restoring a shard, the shard will + // be marked as failed. Either one is a success for the purpose of this test. + try { + RestoreInfo restoreInfo = future.actionGet(); + assertEquals(0, restoreInfo.successfulShards()); + assertEquals(numberOfPrimaryShards, restoreInfo.failedShards()); + } catch (Exception e) { + assertThat(ExceptionsHelper.unwrapCause(e), instanceOf(ElasticsearchTimeoutException.class)); + } + + + for (MockTransportService transportService : transportServices) { + transportService.clearAllRules(); + } + + settingsRequest = new ClusterUpdateSettingsRequest(); + TimeValue defaultValue = CcrSettings.INDICES_RECOVERY_ACTION_TIMEOUT_SETTING.getDefault(Settings.EMPTY); + settingsRequest.persistentSettings(Settings.builder().put(CcrSettings.INDICES_RECOVERY_ACTION_TIMEOUT_SETTING.getKey(), + defaultValue)); + assertAcked(followerClient().admin().cluster().updateSettings(settingsRequest).actionGet()); + } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37887") public void testFollowerMappingIsUpdated() throws IOException { String leaderClusterRepoName = CcrRepository.NAME_PREFIX + "leader_cluster"; diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceServiceTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceServiceTests.java index 5f352788d9597..3035b96b5bcac 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceServiceTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceServiceTests.java @@ -40,7 +40,7 @@ public void setUp() throws Exception { Settings settings = Settings.builder().put(NODE_NAME_SETTING.getKey(), "node").build(); taskQueue = new DeterministicTaskQueue(settings, random()); Set> registeredSettings = Sets.newHashSet(CcrSettings.INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING, - CcrSettings.RECOVERY_MAX_BYTES_PER_SECOND); + CcrSettings.RECOVERY_MAX_BYTES_PER_SECOND, CcrSettings.INDICES_RECOVERY_ACTION_TIMEOUT_SETTING); ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, registeredSettings); restoreSourceService = new CcrRestoreSourceService(taskQueue.getThreadPool(), new CcrSettings(Settings.EMPTY, clusterSettings)); } From 193017672a3a283a2e5bb85f1002f656a4e5e156 Mon Sep 17 00:00:00 2001 From: jimczi Date: Tue, 29 Jan 2019 20:31:24 +0100 Subject: [PATCH 028/100] Handle completion suggestion without contexts This change fixes the handling of completion suggestion without contexts. Relates #36996 --- .../suggest/completion/TopSuggestGroupDocsCollector.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/search/suggest/completion/TopSuggestGroupDocsCollector.java b/server/src/main/java/org/elasticsearch/search/suggest/completion/TopSuggestGroupDocsCollector.java index 3dfb38bef9dd4..30c642dcdeb3b 100644 --- a/server/src/main/java/org/elasticsearch/search/suggest/completion/TopSuggestGroupDocsCollector.java +++ b/server/src/main/java/org/elasticsearch/search/suggest/completion/TopSuggestGroupDocsCollector.java @@ -60,7 +60,9 @@ public void collect(int docID, CharSequence key, CharSequence context, float sco int globalDoc = docID + docBase; boolean isNewDoc = docContexts.containsKey(globalDoc) == false; List contexts = docContexts.computeIfAbsent(globalDoc, k -> new ArrayList<>()); - contexts.add(context); + if (context != null) { + contexts.add(context); + } if (isNewDoc) { super.collect(docID, key, context, score); } From b889221f75048bebbc7084bf3b3f341eac30e34c Mon Sep 17 00:00:00 2001 From: markharwood Date: Tue, 29 Jan 2019 20:52:41 +0000 Subject: [PATCH 029/100] Types removal - deprecate include_type_name with index templates (#37484) Added deprecation warnings for use of include_type_name in put/get index templates. HLRC changes: GetIndexTemplateRequest has a new client-side class which is a copy of server's GetIndexTemplateResponse but modified to be typeless. PutIndexTemplateRequest has a new client-side counterpart which doesn't use types in the mappings Relates to #35190 --- .../elasticsearch/client/IndicesClient.java | 109 ++++- .../client/IndicesRequestConverters.java | 41 +- .../indices/GetIndexTemplatesResponse.java | 90 ++++ .../client/indices/IndexTemplateMetaData.java | 300 ++++++++++++ .../indices/PutIndexTemplateRequest.java | 453 ++++++++++++++++++ .../elasticsearch/client/IndicesClientIT.java | 243 +++++++++- .../client/IndicesRequestConvertersTests.java | 62 ++- .../client/RestHighLevelClientTests.java | 13 +- .../IndicesClientDocumentationIT.java | 74 ++- .../GetIndexTemplatesResponseTests.java | 136 ++++++ .../indices/PutIndexTemplateRequestTests.java | 116 +++++ .../high-level/indices/put_template.asciidoc | 10 +- .../upgrades/FullClusterRestartIT.java | 4 + .../get/GetIndexTemplatesResponse.java | 2 +- .../indices/RestGetIndexTemplateAction.java | 9 + .../indices/RestPutIndexTemplateAction.java | 6 + .../RestPutIndexTemplateActionTests.java | 63 ++- 17 files changed, 1623 insertions(+), 108 deletions(-) create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexTemplatesResponse.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/IndexTemplateMetaData.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/PutIndexTemplateRequest.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexTemplatesResponseTests.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/indices/PutIndexTemplateRequestTests.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java index c93c9c67e8f7f..2f5bd65fba189 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java @@ -49,8 +49,6 @@ import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; import org.elasticsearch.action.admin.indices.shrink.ResizeResponse; import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest; -import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; import org.elasticsearch.action.support.master.AcknowledgedResponse; @@ -61,7 +59,9 @@ import org.elasticsearch.client.indices.GetIndexTemplatesRequest; import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.client.indices.GetMappingsResponse; +import org.elasticsearch.client.indices.GetIndexTemplatesResponse; import org.elasticsearch.client.indices.IndexTemplatesExistRequest; +import org.elasticsearch.client.indices.PutIndexTemplateRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.UnfreezeIndexRequest; import org.elasticsearch.rest.RestStatus; @@ -908,6 +908,7 @@ public void putSettingsAsync(UpdateSettingsRequest updateSettingsRequest, Reques AcknowledgedResponse::fromXContent, listener, emptySet()); } + /** * Puts an index template using the Index Templates API. * See Index Templates API @@ -916,9 +917,13 @@ public void putSettingsAsync(UpdateSettingsRequest updateSettingsRequest, Reques * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @return the response * @throws IOException in case there is a problem sending the request or parsing back the response + * @deprecated This old form of request allows types in mappings. Use {@link #putTemplate(PutIndexTemplateRequest, RequestOptions)} + * instead which introduces a new request object without types. */ - public AcknowledgedResponse putTemplate(PutIndexTemplateRequest putIndexTemplateRequest, - RequestOptions options) throws IOException { + @Deprecated + public AcknowledgedResponse putTemplate( + org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putIndexTemplateRequest, + RequestOptions options) throws IOException { return restHighLevelClient.performRequestAndParseEntity(putIndexTemplateRequest, IndicesRequestConverters::putTemplate, options, AcknowledgedResponse::fromXContent, emptySet()); } @@ -930,9 +935,44 @@ public AcknowledgedResponse putTemplate(PutIndexTemplateRequest putIndexTemplate * @param putIndexTemplateRequest the request * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @param listener the listener to be notified upon request completion + * @deprecated This old form of request allows types in mappings. + * Use {@link #putTemplateAsync(PutIndexTemplateRequest, RequestOptions, ActionListener)} + * instead which introduces a new request object without types. */ - public void putTemplateAsync(PutIndexTemplateRequest putIndexTemplateRequest, RequestOptions options, - ActionListener listener) { + @Deprecated + public void putTemplateAsync(org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putIndexTemplateRequest, + RequestOptions options, ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(putIndexTemplateRequest, IndicesRequestConverters::putTemplate, options, + AcknowledgedResponse::fromXContent, listener, emptySet()); + } + + + /** + * Puts an index template using the Index Templates API. + * See Index Templates API + * on elastic.co + * @param putIndexTemplateRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public AcknowledgedResponse putTemplate( + PutIndexTemplateRequest putIndexTemplateRequest, + RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(putIndexTemplateRequest, IndicesRequestConverters::putTemplate, options, + AcknowledgedResponse::fromXContent, emptySet()); + } + + /** + * Asynchronously puts an index template using the Index Templates API. + * See Index Templates API + * on elastic.co + * @param putIndexTemplateRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void putTemplateAsync(PutIndexTemplateRequest putIndexTemplateRequest, + RequestOptions options, ActionListener listener) { restHighLevelClient.performRequestAsyncAndParseEntity(putIndexTemplateRequest, IndicesRequestConverters::putTemplate, options, AcknowledgedResponse::fromXContent, listener, emptySet()); } @@ -968,33 +1008,74 @@ public void validateQueryAsync(ValidateQueryRequest validateQueryRequest, Reques } /** - * Gets index templates using the Index Templates API + * Gets index templates using the Index Templates API. The mappings will be returned in a legacy deprecated format, where the + * mapping definition is nested under the type name. * See Index Templates API * on elastic.co * @param getIndexTemplatesRequest the request * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @return the response * @throws IOException in case there is a problem sending the request or parsing back the response + * @deprecated This method uses an old response object which still refers to types, a deprecated feature. Use + * {@link #getIndexTemplate(GetIndexTemplatesRequest, RequestOptions)} instead which returns a new response object */ - public GetIndexTemplatesResponse getTemplate(GetIndexTemplatesRequest getIndexTemplatesRequest, - RequestOptions options) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(getIndexTemplatesRequest, IndicesRequestConverters::getTemplates, - options, GetIndexTemplatesResponse::fromXContent, emptySet()); + @Deprecated + public org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse getTemplate( + GetIndexTemplatesRequest getIndexTemplatesRequest, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(getIndexTemplatesRequest, + IndicesRequestConverters::getTemplatesWithDocumentTypes, + options, org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse::fromXContent, emptySet()); } + + /** + * Gets index templates using the Index Templates API + * See Index Templates API + * on elastic.co + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param getIndexTemplatesRequest the request + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public GetIndexTemplatesResponse getIndexTemplate(GetIndexTemplatesRequest getIndexTemplatesRequest, RequestOptions options) + throws IOException { + return restHighLevelClient.performRequestAndParseEntity(getIndexTemplatesRequest, + IndicesRequestConverters::getTemplates, + options, GetIndexTemplatesResponse::fromXContent, emptySet()); + } /** - * Asynchronously gets index templates using the Index Templates API + * Asynchronously gets index templates using the Index Templates API. The mappings will be returned in a legacy deprecated format, + * where the mapping definition is nested under the type name. * See Index Templates API * on elastic.co * @param getIndexTemplatesRequest the request * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @param listener the listener to be notified upon request completion + * @deprecated This method uses an old response object which still refers to types, a deprecated feature. Use + * {@link #getIndexTemplateAsync(GetIndexTemplatesRequest, RequestOptions, ActionListener)} instead which returns a new response object */ + @Deprecated public void getTemplateAsync(GetIndexTemplatesRequest getIndexTemplatesRequest, RequestOptions options, + ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(getIndexTemplatesRequest, + IndicesRequestConverters::getTemplatesWithDocumentTypes, + options, org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse::fromXContent, listener, emptySet()); + } + + /** + * Asynchronously gets index templates using the Index Templates API + * See Index Templates API + * on elastic.co + * @param getIndexTemplatesRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void getIndexTemplateAsync(GetIndexTemplatesRequest getIndexTemplatesRequest, RequestOptions options, ActionListener listener) { - restHighLevelClient.performRequestAsyncAndParseEntity(getIndexTemplatesRequest, IndicesRequestConverters::getTemplates, + restHighLevelClient.performRequestAsyncAndParseEntity(getIndexTemplatesRequest, + IndicesRequestConverters::getTemplates, options, GetIndexTemplatesResponse::fromXContent, listener, emptySet()); - } + } /** * Uses the Index Templates API to determine if index templates exist diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java index 4889ead93b715..13bc2b8c149db 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java @@ -43,13 +43,13 @@ import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.client.indices.FreezeIndexRequest; import org.elasticsearch.client.indices.GetIndexTemplatesRequest; import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.client.indices.IndexTemplatesExistRequest; +import org.elasticsearch.client.indices.PutIndexTemplateRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.UnfreezeIndexRequest; import org.elasticsearch.common.Strings; @@ -416,7 +416,13 @@ static Request indexPutSettings(UpdateSettingsRequest updateSettingsRequest) thr return request; } - static Request putTemplate(PutIndexTemplateRequest putIndexTemplateRequest) throws IOException { + /** + * @deprecated This uses the old form of PutIndexTemplateRequest which uses types. + * Use (@link {@link #putTemplate(PutIndexTemplateRequest)} instead + */ + @Deprecated + static Request putTemplate(org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putIndexTemplateRequest) + throws IOException { String endpoint = new RequestConverters.EndpointBuilder().addPathPartAsIs("_template") .addPathPart(putIndexTemplateRequest.name()).build(); Request request = new Request(HttpPut.METHOD_NAME, endpoint); @@ -433,6 +439,22 @@ static Request putTemplate(PutIndexTemplateRequest putIndexTemplateRequest) thro return request; } + static Request putTemplate(PutIndexTemplateRequest putIndexTemplateRequest) throws IOException { + String endpoint = new RequestConverters.EndpointBuilder().addPathPartAsIs("_template") + .addPathPart(putIndexTemplateRequest.name()).build(); + Request request = new Request(HttpPut.METHOD_NAME, endpoint); + RequestConverters.Params params = new RequestConverters.Params(request); + params.withMasterTimeout(putIndexTemplateRequest.masterNodeTimeout()); + if (putIndexTemplateRequest.create()) { + params.putParam("create", Boolean.TRUE.toString()); + } + if (Strings.hasText(putIndexTemplateRequest.cause())) { + params.putParam("cause", putIndexTemplateRequest.cause()); + } + request.setEntity(RequestConverters.createEntity(putIndexTemplateRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE)); + return request; + } + static Request validateQuery(ValidateQueryRequest validateQueryRequest) throws IOException { String[] indices = validateQueryRequest.indices() == null ? Strings.EMPTY_ARRAY : validateQueryRequest.indices(); String[] types = validateQueryRequest.types() == null || indices.length <= 0 ? Strings.EMPTY_ARRAY : validateQueryRequest.types(); @@ -458,7 +480,16 @@ static Request getAlias(GetAliasesRequest getAliasesRequest) { return request; } + @Deprecated + static Request getTemplatesWithDocumentTypes(GetIndexTemplatesRequest getIndexTemplatesRequest) { + return getTemplates(getIndexTemplatesRequest, true); + } + static Request getTemplates(GetIndexTemplatesRequest getIndexTemplatesRequest) { + return getTemplates(getIndexTemplatesRequest, false); + } + + private static Request getTemplates(GetIndexTemplatesRequest getIndexTemplatesRequest, boolean includeTypeName) { final String endpoint = new RequestConverters.EndpointBuilder() .addPathPartAsIs("_template") .addCommaSeparatedPathParts(getIndexTemplatesRequest.names()) @@ -467,9 +498,11 @@ static Request getTemplates(GetIndexTemplatesRequest getIndexTemplatesRequest) { final RequestConverters.Params params = new RequestConverters.Params(request); params.withLocal(getIndexTemplatesRequest.isLocal()); params.withMasterTimeout(getIndexTemplatesRequest.getMasterNodeTimeout()); - params.putParam(INCLUDE_TYPE_NAME_PARAMETER, "true"); + if (includeTypeName) { + params.putParam(INCLUDE_TYPE_NAME_PARAMETER, "true"); + } return request; - } + } static Request templatesExist(IndexTemplatesExistRequest indexTemplatesExistRequest) { final String endpoint = new RequestConverters.EndpointBuilder() diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexTemplatesResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexTemplatesResponse.java new file mode 100644 index 0000000000000..ef283b31c0634 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexTemplatesResponse.java @@ -0,0 +1,90 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.indices; + +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; + + +public class GetIndexTemplatesResponse { + + @Override + public String toString() { + List thisList = new ArrayList<>(this.indexTemplates); + thisList.sort(Comparator.comparing(IndexTemplateMetaData::name)); + return "GetIndexTemplatesResponse [indexTemplates=" + thisList + "]"; + } + + private final List indexTemplates; + + GetIndexTemplatesResponse() { + indexTemplates = new ArrayList<>(); + } + + GetIndexTemplatesResponse(List indexTemplates) { + this.indexTemplates = indexTemplates; + } + + public List getIndexTemplates() { + return indexTemplates; + } + + + public static GetIndexTemplatesResponse fromXContent(XContentParser parser) throws IOException { + final List templates = new ArrayList<>(); + for (XContentParser.Token token = parser.nextToken(); token != XContentParser.Token.END_OBJECT; token = parser.nextToken()) { + if (token == XContentParser.Token.FIELD_NAME) { + final IndexTemplateMetaData templateMetaData = IndexTemplateMetaData.Builder.fromXContent(parser, parser.currentName()); + templates.add(templateMetaData); + } + } + return new GetIndexTemplatesResponse(templates); + } + + @Override + public int hashCode() { + List sortedList = new ArrayList<>(this.indexTemplates); + sortedList.sort(Comparator.comparing(IndexTemplateMetaData::name)); + return Objects.hash(sortedList); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + // To compare results we need to make sure the templates are listed in the same order + GetIndexTemplatesResponse other = (GetIndexTemplatesResponse) obj; + List thisList = new ArrayList<>(this.indexTemplates); + List otherList = new ArrayList<>(other.indexTemplates); + thisList.sort(Comparator.comparing(IndexTemplateMetaData::name)); + otherList.sort(Comparator.comparing(IndexTemplateMetaData::name)); + return Objects.equals(thisList, otherList); + } + + +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/IndexTemplateMetaData.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/IndexTemplateMetaData.java new file mode 100644 index 0000000000000..12fc747ab3473 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/IndexTemplateMetaData.java @@ -0,0 +1,300 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.indices; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.cluster.metadata.AliasMetaData; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.mapper.MapperService; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +public class IndexTemplateMetaData { + + + private final String name; + + private final int order; + + /** + * The version is an arbitrary number managed by the user so that they can easily and quickly verify the existence of a given template. + * Expected usage: + * 

+     * PUT /_template/my_template
+     * {
+     *   "index_patterns": ["my_index-*"],
+     *   "mappings": { ... },
+     *   "version": 1
+     * }
+     *
+ * Then, some process from the user can occasionally verify that the template exists with the appropriate version without having to + * check the template's content: + * 

+     * GET /_template/my_template?filter_path=*.version
+     *
+ */ + @Nullable + private final Integer version; + + private final List patterns; + + private final Settings settings; + + private final MappingMetaData mappings; + + private final ImmutableOpenMap aliases; + + public IndexTemplateMetaData(String name, int order, Integer version, + List patterns, Settings settings, + MappingMetaData mappings, + ImmutableOpenMap aliases) { + if (patterns == null || patterns.isEmpty()) { + throw new IllegalArgumentException("Index patterns must not be null or empty; got " + patterns); + } + this.name = name; + this.order = order; + this.version = version; + this.patterns= patterns; + this.settings = settings; + this.mappings = mappings; + this.aliases = aliases; + } + + public String name() { + return this.name; + } + + public int order() { + return this.order; + } + + @Nullable + public Integer version() { + return version; + } + + public List patterns() { + return this.patterns; + } + + public Settings settings() { + return this.settings; + } + + public MappingMetaData mappings() { + return this.mappings; + } + + public ImmutableOpenMap aliases() { + return this.aliases; + } + + public static Builder builder(String name) { + return new Builder(name); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + IndexTemplateMetaData that = (IndexTemplateMetaData) o; + + if (order != that.order) return false; + if (!Objects.equals(mappings, that.mappings)) return false; + if (!name.equals(that.name)) return false; + if (!settings.equals(that.settings)) return false; + if (!patterns.equals(that.patterns)) return false; + + return Objects.equals(version, that.version); + } + + @Override + public int hashCode() { + return Objects.hash(name, order, version, patterns, settings, mappings); + } + + public static class Builder { + + private static final Set VALID_FIELDS = Sets.newHashSet( + "template", "order", "mappings", "settings", "index_patterns", "aliases", "version"); + + private String name; + + private int order; + + private Integer version; + + private List indexPatterns; + + private Settings settings = Settings.Builder.EMPTY_SETTINGS; + + private MappingMetaData mappings; + + private final ImmutableOpenMap.Builder aliases; + + public Builder(String name) { + this.name = name; + mappings = null; + aliases = ImmutableOpenMap.builder(); + } + + public Builder(IndexTemplateMetaData indexTemplateMetaData) { + this.name = indexTemplateMetaData.name(); + order(indexTemplateMetaData.order()); + version(indexTemplateMetaData.version()); + patterns(indexTemplateMetaData.patterns()); + settings(indexTemplateMetaData.settings()); + + mappings = indexTemplateMetaData.mappings(); + aliases = ImmutableOpenMap.builder(indexTemplateMetaData.aliases()); + } + + public Builder order(int order) { + this.order = order; + return this; + } + + public Builder version(Integer version) { + this.version = version; + return this; + } + + public Builder patterns(List indexPatterns) { + this.indexPatterns = indexPatterns; + return this; + } + + + public Builder settings(Settings.Builder settings) { + this.settings = settings.build(); + return this; + } + + public Builder settings(Settings settings) { + this.settings = settings; + return this; + } + + public Builder mapping(MappingMetaData mappings) { + this.mappings = mappings; + return this; + } + + public Builder putAlias(AliasMetaData aliasMetaData) { + aliases.put(aliasMetaData.alias(), aliasMetaData); + return this; + } + + public Builder putAlias(AliasMetaData.Builder aliasMetaData) { + aliases.put(aliasMetaData.alias(), aliasMetaData.build()); + return this; + } + + public IndexTemplateMetaData build() { + return new IndexTemplateMetaData(name, order, version, indexPatterns, settings, mappings, aliases.build()); + } + + + public static IndexTemplateMetaData fromXContent(XContentParser parser, String templateName) throws IOException { + Builder builder = new Builder(templateName); + + String currentFieldName = skipTemplateName(parser); + XContentParser.Token token; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_OBJECT) { + if ("settings".equals(currentFieldName)) { + Settings.Builder templateSettingsBuilder = Settings.builder(); + templateSettingsBuilder.put(Settings.fromXContent(parser)); + templateSettingsBuilder.normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX); + builder.settings(templateSettingsBuilder.build()); + } else if ("mappings".equals(currentFieldName)) { + Map mapping = parser.map(); + if (mapping.isEmpty() == false) { + MappingMetaData md = new MappingMetaData(MapperService.SINGLE_MAPPING_NAME, mapping); + builder.mapping(md); + } + } else if ("aliases".equals(currentFieldName)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + builder.putAlias(AliasMetaData.Builder.fromXContent(parser)); + } + } else { + throw new ElasticsearchParseException("unknown key [{}] for index template", currentFieldName); + } + } else if (token == XContentParser.Token.START_ARRAY) { + if ("mappings".equals(currentFieldName)) { + // The server-side IndexTemplateMetaData has toXContent impl that can return mappings + // in an array but also a comment saying this never happens with typeless APIs. + throw new ElasticsearchParseException("Invalid response format - " + + "mappings are not expected to be returned in an array", currentFieldName); + } else if ("index_patterns".equals(currentFieldName)) { + List index_patterns = new ArrayList<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + index_patterns.add(parser.text()); + } + builder.patterns(index_patterns); + } + } else if (token.isValue()) { + // Prior to 5.1.0, elasticsearch only supported a single index pattern called `template` (#21009) + if("template".equals(currentFieldName)) { + builder.patterns(Collections.singletonList(parser.text())); + } else if ("order".equals(currentFieldName)) { + builder.order(parser.intValue()); + } else if ("version".equals(currentFieldName)) { + builder.version(parser.intValue()); + } + } + } + return builder.build(); + } + + private static String skipTemplateName(XContentParser parser) throws IOException { + XContentParser.Token token = parser.nextToken(); + if (token == XContentParser.Token.START_OBJECT) { + token = parser.nextToken(); + if (token == XContentParser.Token.FIELD_NAME) { + String currentFieldName = parser.currentName(); + if (VALID_FIELDS.contains(currentFieldName)) { + return currentFieldName; + } else { + // we just hit the template name, which should be ignored and we move on + parser.nextToken(); + } + } + } + + return null; + } + } + +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/PutIndexTemplateRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/PutIndexTemplateRequest.java new file mode 100644 index 0000000000000..5f22691b046eb --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/PutIndexTemplateRequest.java @@ -0,0 +1,453 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.indices; + +import org.elasticsearch.ElasticsearchGenerationException; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.DeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.common.xcontent.support.XContentMapValues; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.elasticsearch.action.ValidateActions.addValidationError; +import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS; + +/** + * A request to create an index template. + */ +public class PutIndexTemplateRequest extends MasterNodeRequest implements IndicesRequest, ToXContent { + + private String name; + + private String cause = ""; + + private List indexPatterns; + + private int order; + + private boolean create; + + private Settings settings = EMPTY_SETTINGS; + + private BytesReference mappings = null; + + private final Set aliases = new HashSet<>(); + + private Integer version; + + /** + * Constructs a new put index template request with the provided name. + */ + public PutIndexTemplateRequest(String name) { + this.name(name); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (indexPatterns == null || indexPatterns.size() == 0) { + validationException = addValidationError("index patterns are missing", validationException); + } + return validationException; + } + + /** + * Sets the name of the index template. + */ + public PutIndexTemplateRequest name(String name) { + if(name == null) { + throw new IllegalArgumentException("Name cannot be null"); + } + this.name = name; + return this; + } + + /** + * The name of the index template. + */ + public String name() { + return this.name; + } + + public PutIndexTemplateRequest patterns(List indexPatterns) { + this.indexPatterns = indexPatterns; + return this; + } + + public List patterns() { + return this.indexPatterns; + } + + public PutIndexTemplateRequest order(int order) { + this.order = order; + return this; + } + + public int order() { + return this.order; + } + + public PutIndexTemplateRequest version(Integer version) { + this.version = version; + return this; + } + + public Integer version() { + return this.version; + } + + /** + * Set to {@code true} to force only creation, not an update of an index template. If it already + * exists, it will fail with an {@link IllegalArgumentException}. + */ + public PutIndexTemplateRequest create(boolean create) { + this.create = create; + return this; + } + + public boolean create() { + return create; + } + + /** + * The settings to create the index template with. + */ + public PutIndexTemplateRequest settings(Settings settings) { + this.settings = settings; + return this; + } + + /** + * The settings to create the index template with. + */ + public PutIndexTemplateRequest settings(Settings.Builder settings) { + this.settings = settings.build(); + return this; + } + + /** + * The settings to create the index template with (either json/yaml format). + */ + public PutIndexTemplateRequest settings(String source, XContentType xContentType) { + this.settings = Settings.builder().loadFromSource(source, xContentType).build(); + return this; + } + + /** + * The settings to create the index template with (either json or yaml format). + */ + public PutIndexTemplateRequest settings(Map source) { + try { + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + builder.map(source); + settings(Strings.toString(builder), XContentType.JSON); + } catch (IOException e) { + throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); + } + return this; + } + + public Settings settings() { + return this.settings; + } + + /** + * Adds mapping that will be added when the index gets created. + * + * @param source The mapping source + * @param xContentType The type of content contained within the source + */ + public PutIndexTemplateRequest mapping(String source, XContentType xContentType) { + internalMapping(XContentHelper.convertToMap(new BytesArray(source), true, xContentType).v2()); + return this; + } + + /** + * The cause for this index template creation. + */ + public PutIndexTemplateRequest cause(String cause) { + this.cause = cause; + return this; + } + + public String cause() { + return this.cause; + } + + /** + * Adds mapping that will be added when the index gets created. + * + * @param source The mapping source + */ + public PutIndexTemplateRequest mapping(XContentBuilder source) { + internalMapping(XContentHelper.convertToMap(BytesReference.bytes(source), + true, source.contentType()).v2()); + return this; + } + + /** + * Adds mapping that will be added when the index gets created. + * + * @param source The mapping source + * @param xContentType the source content type + */ + public PutIndexTemplateRequest mapping(BytesReference source, XContentType xContentType) { + internalMapping(XContentHelper.convertToMap(source, true, xContentType).v2()); + return this; + } + + /** + * Adds mapping that will be added when the index gets created. + * + * @param source The mapping source + */ + public PutIndexTemplateRequest mapping(Map source) { + return internalMapping(source); + } + + private PutIndexTemplateRequest internalMapping(Map source) { + try { + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + builder.map(source); + Objects.requireNonNull(builder.contentType()); + try { + mappings = new BytesArray( + XContentHelper.convertToJson(BytesReference.bytes(builder), false, false, builder.contentType())); + return this; + } catch (IOException e) { + throw new UncheckedIOException("failed to convert source to json", e); + } + } catch (IOException e) { + throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); + } + } + + public BytesReference mappings() { + return this.mappings; + } + + /** + * The template source definition. + */ + public PutIndexTemplateRequest source(XContentBuilder templateBuilder) { + try { + return source(BytesReference.bytes(templateBuilder), templateBuilder.contentType()); + } catch (Exception e) { + throw new IllegalArgumentException("Failed to build json for template request", e); + } + } + + /** + * The template source definition. + */ + @SuppressWarnings("unchecked") + public PutIndexTemplateRequest source(Map templateSource) { + Map source = templateSource; + for (Map.Entry entry : source.entrySet()) { + String name = entry.getKey(); + if (name.equals("template")) { + if(entry.getValue() instanceof String) { + patterns(Collections.singletonList((String) entry.getValue())); + } + } else if (name.equals("index_patterns")) { + if(entry.getValue() instanceof String) { + patterns(Collections.singletonList((String) entry.getValue())); + } else if (entry.getValue() instanceof List) { + List elements = ((List) entry.getValue()).stream().map(Object::toString).collect(Collectors.toList()); + patterns(elements); + } else { + throw new IllegalArgumentException("Malformed [template] value, should be a string or a list of strings"); + } + } else if (name.equals("order")) { + order(XContentMapValues.nodeIntegerValue(entry.getValue(), order())); + } else if ("version".equals(name)) { + if ((entry.getValue() instanceof Integer) == false) { + throw new IllegalArgumentException("Malformed [version] value, should be an integer"); + } + version((Integer)entry.getValue()); + } else if (name.equals("settings")) { + if ((entry.getValue() instanceof Map) == false) { + throw new IllegalArgumentException("Malformed [settings] section, should include an inner object"); + } + settings((Map) entry.getValue()); + } else if (name.equals("mappings")) { + Map mappings = (Map) entry.getValue(); + mapping(mappings); + } else if (name.equals("aliases")) { + aliases((Map) entry.getValue()); + } else { + throw new ElasticsearchParseException("unknown key [{}] in the template ", name); + } + } + return this; + } + + /** + * The template source definition. + */ + public PutIndexTemplateRequest source(String templateSource, XContentType xContentType) { + return source(XContentHelper.convertToMap(xContentType.xContent(), templateSource, true)); + } + + /** + * The template source definition. + */ + public PutIndexTemplateRequest source(byte[] source, XContentType xContentType) { + return source(source, 0, source.length, xContentType); + } + + /** + * The template source definition. + */ + public PutIndexTemplateRequest source(byte[] source, int offset, int length, XContentType xContentType) { + return source(new BytesArray(source, offset, length), xContentType); + } + + /** + * The template source definition. + */ + public PutIndexTemplateRequest source(BytesReference source, XContentType xContentType) { + return source(XContentHelper.convertToMap(source, true, xContentType).v2()); + } + + + public Set aliases() { + return this.aliases; + } + + /** + * Sets the aliases that will be associated with the index when it gets created + */ + public PutIndexTemplateRequest aliases(Map source) { + try { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.map(source); + return aliases(BytesReference.bytes(builder)); + } catch (IOException e) { + throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); + } + } + + /** + * Sets the aliases that will be associated with the index when it gets created + */ + public PutIndexTemplateRequest aliases(XContentBuilder source) { + return aliases(BytesReference.bytes(source)); + } + + /** + * Sets the aliases that will be associated with the index when it gets created + */ + public PutIndexTemplateRequest aliases(String source) { + return aliases(new BytesArray(source)); + } + + /** + * Sets the aliases that will be associated with the index when it gets created + */ + public PutIndexTemplateRequest aliases(BytesReference source) { + // EMPTY is safe here because we never call namedObject + try (XContentParser parser = XContentHelper + .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, source)) { + //move to the first alias + parser.nextToken(); + while ((parser.nextToken()) != XContentParser.Token.END_OBJECT) { + alias(Alias.fromXContent(parser)); + } + return this; + } catch(IOException e) { + throw new ElasticsearchParseException("Failed to parse aliases", e); + } + } + + /** + * Adds an alias that will be added when the index gets created. + * + * @param alias The metadata for the new alias + * @return the index template creation request + */ + public PutIndexTemplateRequest alias(Alias alias) { + aliases.add(alias); + return this; + } + + @Override + public String[] indices() { + return indexPatterns.toArray(new String[indexPatterns.size()]); + } + + @Override + public IndicesOptions indicesOptions() { + return IndicesOptions.strictExpand(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field("index_patterns", indexPatterns); + builder.field("order", order); + if (version != null) { + builder.field("version", version); + } + + builder.startObject("settings"); + settings.toXContent(builder, params); + builder.endObject(); + + if (mappings != null) { + builder.field("mappings"); + try (XContentParser parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mappings.utf8ToString())) { + builder.copyCurrentStructure(parser); + } + } + + builder.startObject("aliases"); + for (Alias alias : aliases) { + alias.toXContent(builder, params); + } + builder.endObject(); + + return builder; + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java index 708cb6687ce9e..306929d78a67a 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java @@ -54,8 +54,6 @@ import org.elasticsearch.action.admin.indices.shrink.ResizeResponse; import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest; -import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; import org.elasticsearch.action.index.IndexRequest; @@ -72,12 +70,14 @@ import org.elasticsearch.client.indices.GetIndexTemplatesRequest; import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.client.indices.GetMappingsResponse; +import org.elasticsearch.client.indices.GetIndexTemplatesResponse; +import org.elasticsearch.client.indices.IndexTemplateMetaData; import org.elasticsearch.client.indices.IndexTemplatesExistRequest; +import org.elasticsearch.client.indices.PutIndexTemplateRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.UnfreezeIndexRequest; import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Setting; @@ -86,6 +86,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.IndexSettings; @@ -97,6 +98,8 @@ import org.elasticsearch.rest.action.admin.indices.RestGetFieldMappingAction; import org.elasticsearch.rest.action.admin.indices.RestGetMappingAction; import org.elasticsearch.rest.action.admin.indices.RestPutMappingAction; +import org.elasticsearch.rest.action.admin.indices.RestGetIndexTemplateAction; +import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction; import java.io.IOException; import java.util.Arrays; @@ -1400,8 +1403,9 @@ public void testIndexPutSettingNonExistent() throws IOException { } @SuppressWarnings("unchecked") - public void testPutTemplate() throws Exception { - PutIndexTemplateRequest putTemplateRequest = new PutIndexTemplateRequest() + public void testPutTemplateWithTypes() throws Exception { + org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putTemplateRequest = + new org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest() .name("my-template") .patterns(Arrays.asList("pattern-1", "name-*")) .order(10) @@ -1411,7 +1415,9 @@ public void testPutTemplate() throws Exception { .alias(new Alias("alias-1").indexRouting("abc")).alias(new Alias("{index}-write").searchRouting("xyz")); AcknowledgedResponse putTemplateResponse = execute(putTemplateRequest, - highLevelClient().indices()::putTemplate, highLevelClient().indices()::putTemplateAsync); + highLevelClient().indices()::putTemplate, highLevelClient().indices()::putTemplateAsync, + expectWarnings(RestPutIndexTemplateAction.TYPES_DEPRECATION_MESSAGE) + ); assertThat(putTemplateResponse.isAcknowledged(), equalTo(true)); Map templates = getAsMap("/_template/my-template"); @@ -1425,6 +1431,94 @@ public void testPutTemplate() throws Exception { assertThat((Map) extractValue("my-template.aliases.alias-1", templates), hasEntry("index_routing", "abc")); assertThat((Map) extractValue("my-template.aliases.{index}-write", templates), hasEntry("search_routing", "xyz")); } + + @SuppressWarnings("unchecked") + public void testPutTemplate() throws Exception { + PutIndexTemplateRequest putTemplateRequest = new PutIndexTemplateRequest("my-template") + .patterns(Arrays.asList("pattern-1", "name-*")) + .order(10) + .create(randomBoolean()) + .settings(Settings.builder().put("number_of_shards", "3").put("number_of_replicas", "0")) + .mapping("{ \"properties\":{" + + "\"host_name\": {\"type\":\"keyword\"}" + + "}" + + "}", XContentType.JSON) + .alias(new Alias("alias-1").indexRouting("abc")).alias(new Alias("{index}-write").searchRouting("xyz")); + + AcknowledgedResponse putTemplateResponse = execute(putTemplateRequest, + highLevelClient().indices()::putTemplate, highLevelClient().indices()::putTemplateAsync); + assertThat(putTemplateResponse.isAcknowledged(), equalTo(true)); + + Map templates = getAsMap("/_template/my-template"); + assertThat(templates.keySet(), hasSize(1)); + assertThat(extractValue("my-template.order", templates), equalTo(10)); + assertThat(extractRawValues("my-template.index_patterns", templates), contains("pattern-1", "name-*")); + assertThat(extractValue("my-template.settings.index.number_of_shards", templates), equalTo("3")); + assertThat(extractValue("my-template.settings.index.number_of_replicas", templates), equalTo("0")); + assertThat(extractValue("my-template.mappings.properties.host_name.type", templates), equalTo("keyword")); + assertThat((Map) extractValue("my-template.aliases.alias-1", templates), hasEntry("index_routing", "abc")); + assertThat((Map) extractValue("my-template.aliases.{index}-write", templates), hasEntry("search_routing", "xyz")); + } + + public void testPutTemplateWithTypesUsingUntypedAPI() throws Exception { + PutIndexTemplateRequest putTemplateRequest = new PutIndexTemplateRequest("my-template") + .patterns(Arrays.asList("pattern-1", "name-*")) + .order(10) + .create(randomBoolean()) + .settings(Settings.builder().put("number_of_shards", "3").put("number_of_replicas", "0")) + .mapping("{ " + + "\"my_doc_type\":{" + + "\"properties\":{" + + "\"host_name\": {\"type\":\"keyword\"}" + + "}" + + "}" + + "}", XContentType.JSON) + .alias(new Alias("alias-1").indexRouting("abc")).alias(new Alias("{index}-write").searchRouting("xyz")); + + + ElasticsearchStatusException badMappingError = expectThrows(ElasticsearchStatusException.class, + () -> execute(putTemplateRequest, + highLevelClient().indices()::putTemplate, highLevelClient().indices()::putTemplateAsync)); + assertThat(badMappingError.getDetailedMessage(), + containsString("Root mapping definition has unsupported parameters: [my_doc_type")); + } + + @SuppressWarnings("unchecked") + public void testPutTemplateWithNoTypesUsingTypedApi() throws Exception { + org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putTemplateRequest = + new org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest() + .name("my-template") + .patterns(Arrays.asList("pattern-1", "name-*")) + .order(10) + .create(randomBoolean()) + .settings(Settings.builder().put("number_of_shards", "3").put("number_of_replicas", "0")) + .mapping("my_doc_type", + // Note that the declared type is missing from the mapping + "{ " + + "\"properties\":{" + + "\"host_name\": {\"type\":\"keyword\"}," + + "\"description\": {\"type\":\"text\"}" + + "}" + + "}", XContentType.JSON) + .alias(new Alias("alias-1").indexRouting("abc")).alias(new Alias("{index}-write").searchRouting("xyz")); + + AcknowledgedResponse putTemplateResponse = execute(putTemplateRequest, + highLevelClient().indices()::putTemplate, highLevelClient().indices()::putTemplateAsync, + expectWarnings(RestPutIndexTemplateAction.TYPES_DEPRECATION_MESSAGE) + ); + assertThat(putTemplateResponse.isAcknowledged(), equalTo(true)); + + Map templates = getAsMap("/_template/my-template"); + assertThat(templates.keySet(), hasSize(1)); + assertThat(extractValue("my-template.order", templates), equalTo(10)); + assertThat(extractRawValues("my-template.index_patterns", templates), contains("pattern-1", "name-*")); + assertThat(extractValue("my-template.settings.index.number_of_shards", templates), equalTo("3")); + assertThat(extractValue("my-template.settings.index.number_of_replicas", templates), equalTo("0")); + assertThat(extractValue("my-template.mappings.properties.host_name.type", templates), equalTo("keyword")); + assertThat(extractValue("my-template.mappings.properties.description.type", templates), equalTo("text")); + assertThat((Map) extractValue("my-template.aliases.alias-1", templates), hasEntry("index_routing", "abc")); + assertThat((Map) extractValue("my-template.aliases.{index}-write", templates), hasEntry("search_routing", "xyz")); + } public void testPutTemplateBadRequests() throws Exception { RestHighLevelClient client = highLevelClient(); @@ -1489,68 +1583,168 @@ public void testInvalidValidateQuery() throws IOException{ assertFalse(response.isValid()); } + // Tests the deprecated form of the API that returns templates with doc types (using the server-side's GetIndexTemplateResponse) + public void testCRUDIndexTemplateWithTypes() throws Exception { + RestHighLevelClient client = highLevelClient(); + + org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putTemplate1 = + new org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest().name("template-1") + .patterns(Arrays.asList("pattern-1", "name-1")).alias(new Alias("alias-1")); + assertThat(execute(putTemplate1, client.indices()::putTemplate, client.indices()::putTemplateAsync + , expectWarnings(RestPutIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)) + .isAcknowledged(), equalTo(true)); + org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putTemplate2 = + new org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest().name("template-2") + .patterns(Arrays.asList("pattern-2", "name-2")) + .mapping("custom_doc_type", "name", "type=text") + .settings(Settings.builder().put("number_of_shards", "2").put("number_of_replicas", "0")); + assertThat(execute(putTemplate2, client.indices()::putTemplate, client.indices()::putTemplateAsync, + expectWarnings(RestPutIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)) + .isAcknowledged(), equalTo(true)); + + org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse getTemplate1 = execute( + new GetIndexTemplatesRequest("template-1"), + client.indices()::getTemplate, client.indices()::getTemplateAsync, + expectWarnings(RestGetIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)); + assertThat(getTemplate1.getIndexTemplates(), hasSize(1)); + org.elasticsearch.cluster.metadata.IndexTemplateMetaData template1 = getTemplate1.getIndexTemplates().get(0); + assertThat(template1.name(), equalTo("template-1")); + assertThat(template1.patterns(), contains("pattern-1", "name-1")); + assertTrue(template1.aliases().containsKey("alias-1")); + + //Check the typed version of the call + org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse getTemplate2 = + execute(new GetIndexTemplatesRequest("template-2"), + client.indices()::getTemplate, client.indices()::getTemplateAsync, + expectWarnings(RestGetIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)); + assertThat(getTemplate2.getIndexTemplates(), hasSize(1)); + org.elasticsearch.cluster.metadata.IndexTemplateMetaData template2 = getTemplate2.getIndexTemplates().get(0); + assertThat(template2.name(), equalTo("template-2")); + assertThat(template2.patterns(), contains("pattern-2", "name-2")); + assertTrue(template2.aliases().isEmpty()); + assertThat(template2.settings().get("index.number_of_shards"), equalTo("2")); + assertThat(template2.settings().get("index.number_of_replicas"), equalTo("0")); + // Ugly deprecated form of API requires use of doc type to get at mapping object which is CompressedXContent + assertTrue(template2.mappings().containsKey("custom_doc_type")); + + List names = randomBoolean() + ? Arrays.asList("*-1", "template-2") + : Arrays.asList("template-*"); + GetIndexTemplatesRequest getBothRequest = new GetIndexTemplatesRequest(names); + org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse getBoth = execute( + getBothRequest, client.indices()::getTemplate, client.indices()::getTemplateAsync, + expectWarnings(RestGetIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)); + assertThat(getBoth.getIndexTemplates(), hasSize(2)); + assertThat(getBoth.getIndexTemplates().stream().map(org.elasticsearch.cluster.metadata.IndexTemplateMetaData::getName).toArray(), + arrayContainingInAnyOrder("template-1", "template-2")); + + GetIndexTemplatesRequest getAllRequest = new GetIndexTemplatesRequest(); + org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse getAll = execute( + getAllRequest, client.indices()::getTemplate, client.indices()::getTemplateAsync, + expectWarnings(RestGetIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)); + assertThat(getAll.getIndexTemplates().size(), greaterThanOrEqualTo(2)); + assertThat(getAll.getIndexTemplates().stream().map(org.elasticsearch.cluster.metadata.IndexTemplateMetaData::getName) + .collect(Collectors.toList()), + hasItems("template-1", "template-2")); + + assertTrue(execute(new DeleteIndexTemplateRequest("template-1"), + client.indices()::deleteTemplate, client.indices()::deleteTemplateAsync).isAcknowledged()); + assertThat(expectThrows(ElasticsearchException.class, () -> execute(new GetIndexTemplatesRequest("template-1"), + client.indices()::getTemplate, client.indices()::getTemplateAsync)).status(), equalTo(RestStatus.NOT_FOUND)); + assertThat(expectThrows(ElasticsearchException.class, () -> execute(new DeleteIndexTemplateRequest("template-1"), + client.indices()::deleteTemplate, client.indices()::deleteTemplateAsync)).status(), equalTo(RestStatus.NOT_FOUND)); + + assertThat(execute(new GetIndexTemplatesRequest("template-*"), + client.indices()::getTemplate, client.indices()::getTemplateAsync, + expectWarnings(RestGetIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)).getIndexTemplates(), hasSize(1)); + assertThat(execute(new GetIndexTemplatesRequest("template-*"), + client.indices()::getTemplate, client.indices()::getTemplateAsync, + expectWarnings(RestGetIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)).getIndexTemplates() + .get(0).name(), equalTo("template-2")); + + assertTrue(execute(new DeleteIndexTemplateRequest("template-*"), + client.indices()::deleteTemplate, client.indices()::deleteTemplateAsync).isAcknowledged()); + assertThat(expectThrows(ElasticsearchException.class, () -> execute(new GetIndexTemplatesRequest("template-*"), + client.indices()::getTemplate, client.indices()::getTemplateAsync, + expectWarnings(RestGetIndexTemplateAction.TYPES_DEPRECATION_MESSAGE))).status(), equalTo(RestStatus.NOT_FOUND)); + } + + public void testCRUDIndexTemplate() throws Exception { RestHighLevelClient client = highLevelClient(); - PutIndexTemplateRequest putTemplate1 = new PutIndexTemplateRequest().name("template-1") + PutIndexTemplateRequest putTemplate1 = new PutIndexTemplateRequest("template-1") .patterns(Arrays.asList("pattern-1", "name-1")).alias(new Alias("alias-1")); assertThat(execute(putTemplate1, client.indices()::putTemplate, client.indices()::putTemplateAsync).isAcknowledged(), equalTo(true)); - PutIndexTemplateRequest putTemplate2 = new PutIndexTemplateRequest().name("template-2") + PutIndexTemplateRequest putTemplate2 = new PutIndexTemplateRequest("template-2") .patterns(Arrays.asList("pattern-2", "name-2")) + .mapping("{\"properties\": { \"name\": { \"type\": \"text\" }}}", XContentType.JSON) .settings(Settings.builder().put("number_of_shards", "2").put("number_of_replicas", "0")); - assertThat(execute(putTemplate2, client.indices()::putTemplate, client.indices()::putTemplateAsync).isAcknowledged(), - equalTo(true)); + assertThat(execute(putTemplate2, client.indices()::putTemplate, client.indices()::putTemplateAsync) + .isAcknowledged(), equalTo(true)); - GetIndexTemplatesResponse getTemplate1 = execute(new GetIndexTemplatesRequest("template-1"), - client.indices()::getTemplate, client.indices()::getTemplateAsync); + GetIndexTemplatesResponse getTemplate1 = execute( + new GetIndexTemplatesRequest("template-1"), + client.indices()::getIndexTemplate, client.indices()::getIndexTemplateAsync); assertThat(getTemplate1.getIndexTemplates(), hasSize(1)); IndexTemplateMetaData template1 = getTemplate1.getIndexTemplates().get(0); assertThat(template1.name(), equalTo("template-1")); assertThat(template1.patterns(), contains("pattern-1", "name-1")); assertTrue(template1.aliases().containsKey("alias-1")); - + GetIndexTemplatesResponse getTemplate2 = execute(new GetIndexTemplatesRequest("template-2"), - client.indices()::getTemplate, client.indices()::getTemplateAsync); + client.indices()::getIndexTemplate, client.indices()::getIndexTemplateAsync); assertThat(getTemplate2.getIndexTemplates(), hasSize(1)); IndexTemplateMetaData template2 = getTemplate2.getIndexTemplates().get(0); assertThat(template2.name(), equalTo("template-2")); assertThat(template2.patterns(), contains("pattern-2", "name-2")); assertTrue(template2.aliases().isEmpty()); assertThat(template2.settings().get("index.number_of_shards"), equalTo("2")); - assertThat(template2.settings().get("index.number_of_replicas"), equalTo("0")); + assertThat(template2.settings().get("index.number_of_replicas"), equalTo("0")); + // New API returns a MappingMetaData class rather than CompressedXContent for the mapping + assertTrue(template2.mappings().sourceAsMap().containsKey("properties")); + @SuppressWarnings("unchecked") + Map props = (Map) template2.mappings().sourceAsMap().get("properties"); + assertTrue(props.containsKey("name")); + + List names = randomBoolean() ? Arrays.asList("*-1", "template-2") : Arrays.asList("template-*"); GetIndexTemplatesRequest getBothRequest = new GetIndexTemplatesRequest(names); - GetIndexTemplatesResponse getBoth = execute(getBothRequest, client.indices()::getTemplate, client.indices()::getTemplateAsync); + GetIndexTemplatesResponse getBoth = execute( + getBothRequest, client.indices()::getIndexTemplate, client.indices()::getIndexTemplateAsync); assertThat(getBoth.getIndexTemplates(), hasSize(2)); - assertThat(getBoth.getIndexTemplates().stream().map(IndexTemplateMetaData::getName).toArray(), + assertThat(getBoth.getIndexTemplates().stream().map(IndexTemplateMetaData::name).toArray(), arrayContainingInAnyOrder("template-1", "template-2")); GetIndexTemplatesRequest getAllRequest = new GetIndexTemplatesRequest(); - GetIndexTemplatesResponse getAll = execute(getAllRequest, client.indices()::getTemplate, client.indices()::getTemplateAsync); + GetIndexTemplatesResponse getAll = execute( + getAllRequest, client.indices()::getIndexTemplate, client.indices()::getIndexTemplateAsync); assertThat(getAll.getIndexTemplates().size(), greaterThanOrEqualTo(2)); - assertThat(getAll.getIndexTemplates().stream().map(IndexTemplateMetaData::getName).collect(Collectors.toList()), + assertThat(getAll.getIndexTemplates().stream().map(IndexTemplateMetaData::name) + .collect(Collectors.toList()), hasItems("template-1", "template-2")); assertTrue(execute(new DeleteIndexTemplateRequest("template-1"), client.indices()::deleteTemplate, client.indices()::deleteTemplateAsync).isAcknowledged()); assertThat(expectThrows(ElasticsearchException.class, () -> execute(new GetIndexTemplatesRequest("template-1"), - client.indices()::getTemplate, client.indices()::getTemplateAsync)).status(), equalTo(RestStatus.NOT_FOUND)); + client.indices()::getIndexTemplate, client.indices()::getIndexTemplateAsync)).status(), equalTo(RestStatus.NOT_FOUND)); assertThat(expectThrows(ElasticsearchException.class, () -> execute(new DeleteIndexTemplateRequest("template-1"), client.indices()::deleteTemplate, client.indices()::deleteTemplateAsync)).status(), equalTo(RestStatus.NOT_FOUND)); assertThat(execute(new GetIndexTemplatesRequest("template-*"), - client.indices()::getTemplate, client.indices()::getTemplateAsync).getIndexTemplates(), hasSize(1)); + client.indices()::getIndexTemplate, client.indices()::getIndexTemplateAsync).getIndexTemplates(), hasSize(1)); assertThat(execute(new GetIndexTemplatesRequest("template-*"), - client.indices()::getTemplate, client.indices()::getTemplateAsync).getIndexTemplates().get(0).name(), equalTo("template-2")); + client.indices()::getIndexTemplate, client.indices()::getIndexTemplateAsync).getIndexTemplates() + .get(0).name(), equalTo("template-2")); assertTrue(execute(new DeleteIndexTemplateRequest("template-*"), client.indices()::deleteTemplate, client.indices()::deleteTemplateAsync).isAcknowledged()); assertThat(expectThrows(ElasticsearchException.class, () -> execute(new GetIndexTemplatesRequest("template-*"), - client.indices()::getTemplate, client.indices()::getTemplateAsync)).status(), equalTo(RestStatus.NOT_FOUND)); + client.indices()::getIndexTemplate, client.indices()::getIndexTemplateAsync)).status(), equalTo(RestStatus.NOT_FOUND)); } public void testIndexTemplatesExist() throws Exception { @@ -1559,8 +1753,7 @@ public void testIndexTemplatesExist() throws Exception { { for (String suffix : Arrays.asList("1", "2")) { - final PutIndexTemplateRequest putRequest = new PutIndexTemplateRequest() - .name("template-" + suffix) + final PutIndexTemplateRequest putRequest = new PutIndexTemplateRequest("template-" + suffix) .patterns(Arrays.asList("pattern-" + suffix, "name-" + suffix)) .alias(new Alias("alias-" + suffix)); assertTrue(execute(putRequest, client.indices()::putTemplate, client.indices()::putTemplateAsync).isAcknowledged()); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java index 409690eaa6997..6d873ec2b944c 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java @@ -45,7 +45,6 @@ import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.client.indices.CreateIndexRequest; @@ -53,6 +52,7 @@ import org.elasticsearch.client.indices.GetIndexTemplatesRequest; import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.client.indices.IndexTemplatesExistRequest; +import org.elasticsearch.client.indices.PutIndexTemplateRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.RandomCreateIndexGenerator; import org.elasticsearch.common.CheckedFunction; @@ -60,6 +60,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import org.junit.Assert; @@ -921,14 +922,16 @@ public void testIndexPutSettings() throws IOException { Assert.assertEquals(expectedParams, request.getParameters()); } - public void testPutTemplateRequest() throws Exception { + public void testPutTemplateRequestWithTypes() throws Exception { Map names = new HashMap<>(); names.put("log", "log"); names.put("template#1", "template%231"); names.put("-#template", "-%23template"); names.put("foo^bar", "foo%5Ebar"); - PutIndexTemplateRequest putTemplateRequest = new PutIndexTemplateRequest().name(ESTestCase.randomFrom(names.keySet())) + org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putTemplateRequest = + new org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest() + .name(ESTestCase.randomFrom(names.keySet())) .patterns(Arrays.asList(ESTestCase.generateRandomStringArray(20, 100, false, false))); if (ESTestCase.randomBoolean()) { putTemplateRequest.order(ESTestCase.randomInt()); @@ -939,14 +942,15 @@ public void testPutTemplateRequest() throws Exception { if (ESTestCase.randomBoolean()) { putTemplateRequest.settings(Settings.builder().put("setting-" + ESTestCase.randomInt(), ESTestCase.randomTimeValue())); } + Map expectedParams = new HashMap<>(); if (ESTestCase.randomBoolean()) { putTemplateRequest.mapping("doc-" + ESTestCase.randomInt(), "field-" + ESTestCase.randomInt(), "type=" + ESTestCase.randomFrom("text", "keyword")); } + expectedParams.put(INCLUDE_TYPE_NAME_PARAMETER, "true"); if (ESTestCase.randomBoolean()) { putTemplateRequest.alias(new Alias("alias-" + ESTestCase.randomInt())); } - Map expectedParams = new HashMap<>(); if (ESTestCase.randomBoolean()) { expectedParams.put("create", Boolean.TRUE.toString()); putTemplateRequest.create(true); @@ -955,9 +959,8 @@ public void testPutTemplateRequest() throws Exception { String cause = ESTestCase.randomUnicodeOfCodepointLengthBetween(1, 50); putTemplateRequest.cause(cause); expectedParams.put("cause", cause); - } + } RequestConvertersTests.setRandomMasterTimeout(putTemplateRequest, expectedParams); - expectedParams.put(INCLUDE_TYPE_NAME_PARAMETER, "true"); Request request = IndicesRequestConverters.putTemplate(putTemplateRequest); Assert.assertThat(request.getEndpoint(), equalTo("/_template/" + names.get(putTemplateRequest.name()))); @@ -965,6 +968,49 @@ public void testPutTemplateRequest() throws Exception { RequestConvertersTests.assertToXContentBody(putTemplateRequest, request.getEntity()); } + public void testPutTemplateRequest() throws Exception { + Map names = new HashMap<>(); + names.put("log", "log"); + names.put("template#1", "template%231"); + names.put("-#template", "-%23template"); + names.put("foo^bar", "foo%5Ebar"); + + PutIndexTemplateRequest putTemplateRequest = + new PutIndexTemplateRequest(ESTestCase.randomFrom(names.keySet())) + .patterns(Arrays.asList(ESTestCase.generateRandomStringArray(20, 100, false, false))); + if (ESTestCase.randomBoolean()) { + putTemplateRequest.order(ESTestCase.randomInt()); + } + if (ESTestCase.randomBoolean()) { + putTemplateRequest.version(ESTestCase.randomInt()); + } + if (ESTestCase.randomBoolean()) { + putTemplateRequest.settings(Settings.builder().put("setting-" + ESTestCase.randomInt(), ESTestCase.randomTimeValue())); + } + Map expectedParams = new HashMap<>(); + if (ESTestCase.randomBoolean()) { + putTemplateRequest.mapping("{ \"properties\": { \"field-" + ESTestCase.randomInt() + + "\" : { \"type\" : \"" + ESTestCase.randomFrom("text", "keyword") + "\" }}}", XContentType.JSON); + } + if (ESTestCase.randomBoolean()) { + putTemplateRequest.alias(new Alias("alias-" + ESTestCase.randomInt())); + } + if (ESTestCase.randomBoolean()) { + expectedParams.put("create", Boolean.TRUE.toString()); + putTemplateRequest.create(true); + } + if (ESTestCase.randomBoolean()) { + String cause = ESTestCase.randomUnicodeOfCodepointLengthBetween(1, 50); + putTemplateRequest.cause(cause); + expectedParams.put("cause", cause); + } + RequestConvertersTests.setRandomMasterTimeout(putTemplateRequest, expectedParams); + + Request request = IndicesRequestConverters.putTemplate(putTemplateRequest); + Assert.assertThat(request.getEndpoint(), equalTo("/_template/" + names.get(putTemplateRequest.name()))); + Assert.assertThat(request.getParameters(), equalTo(expectedParams)); + RequestConvertersTests.assertToXContentBody(putTemplateRequest, request.getEntity()); + } public void testValidateQuery() throws Exception { String[] indices = ESTestCase.randomBoolean() ? null : RequestConvertersTests.randomIndicesNames(0, 5); String[] types = ESTestCase.randomBoolean() ? ESTestCase.generateRandomStringArray(5, 5, false, false) : null; @@ -1012,9 +1058,9 @@ public void testGetTemplateRequest() throws Exception { Map expectedParams = new HashMap<>(); RequestConvertersTests.setRandomMasterTimeout(getTemplatesRequest::setMasterNodeTimeout, expectedParams); RequestConvertersTests.setRandomLocal(getTemplatesRequest::setLocal, expectedParams); - expectedParams.put(INCLUDE_TYPE_NAME_PARAMETER, "true"); - Request request = IndicesRequestConverters.getTemplates(getTemplatesRequest); + Request request = IndicesRequestConverters.getTemplatesWithDocumentTypes(getTemplatesRequest); + expectedParams.put(INCLUDE_TYPE_NAME_PARAMETER, "true"); Assert.assertThat(request.getEndpoint(), equalTo("/_template/" + names.stream().map(encodes::get).collect(Collectors.joining(",")))); Assert.assertThat(request.getParameters(), equalTo(expectedParams)); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java index a20d78f939da5..945c1df19efa3 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java @@ -744,6 +744,14 @@ public void testApiNamingConventions() throws Exception { .collect(Collectors.groupingBy(Tuple::v1, Collectors.mapping(Tuple::v2, Collectors.toSet()))); + // TODO remove in 8.0 - we will undeprecate indices.get_template because the current getIndexTemplate + // impl will replace the existing getTemplate method. + // The above general-purpose code ignores all deprecated methods which in this case leaves `getTemplate` + // looking like it doesn't have a valid implementatation when it does. + apiUnsupported.remove("indices.get_template"); + + + for (Map.Entry> entry : methods.entrySet()) { String apiName = entry.getKey(); @@ -776,7 +784,10 @@ public void testApiNamingConventions() throws Exception { apiName.startsWith("security.") == false && apiName.startsWith("index_lifecycle.") == false && apiName.startsWith("ccr.") == false && - apiName.endsWith("freeze") == false) { + apiName.endsWith("freeze") == false && + // IndicesClientIT.getIndexTemplate should be renamed "getTemplate" in version 8.0 when we + // can get rid of 7.0's deprecated "getTemplate" + apiName.equals("indices.get_index_template") == false) { apiNotFound.add(apiName); } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java index cfd5ac4de2f82..83244b90b950f 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java @@ -55,8 +55,6 @@ import org.elasticsearch.action.admin.indices.shrink.ResizeResponse; import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest; -import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.admin.indices.validate.query.QueryExplanation; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; @@ -76,11 +74,13 @@ import org.elasticsearch.client.indices.GetIndexTemplatesRequest; import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.client.indices.GetMappingsResponse; +import org.elasticsearch.client.indices.GetIndexTemplatesResponse; +import org.elasticsearch.client.indices.IndexTemplateMetaData; import org.elasticsearch.client.indices.IndexTemplatesExistRequest; +import org.elasticsearch.client.indices.PutIndexTemplateRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.UnfreezeIndexRequest; import org.elasticsearch.cluster.metadata.AliasMetaData; -import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; @@ -2094,13 +2094,11 @@ public void testPutTemplate() throws Exception { { // tag::put-template-request-mappings-json - request.mapping("_doc", // <1> + request.mapping(// <1> "{\n" + - " \"_doc\": {\n" + - " \"properties\": {\n" + - " \"message\": {\n" + - " \"type\": \"text\"\n" + - " }\n" + + " \"properties\": {\n" + + " \"message\": {\n" + + " \"type\": \"text\"\n" + " }\n" + " }\n" + "}", // <2> @@ -2111,14 +2109,16 @@ public void testPutTemplate() throws Exception { { //tag::put-template-request-mappings-map Map jsonMap = new HashMap<>(); - Map message = new HashMap<>(); - message.put("type", "text"); - Map properties = new HashMap<>(); - properties.put("message", message); - Map mapping = new HashMap<>(); - mapping.put("properties", properties); - jsonMap.put("_doc", mapping); - request.mapping("_doc", jsonMap); // <1> + { + Map properties = new HashMap<>(); + { + Map message = new HashMap<>(); + message.put("type", "text"); + properties.put("message", message); + } + jsonMap.put("properties", properties); + } + request.mapping(jsonMap); // <1> //end::put-template-request-mappings-map assertTrue(client.indices().putTemplate(request, RequestOptions.DEFAULT).isAcknowledged()); } @@ -2127,31 +2127,21 @@ public void testPutTemplate() throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject(); { - builder.startObject("_doc"); + builder.startObject("properties"); { - builder.startObject("properties"); + builder.startObject("message"); { - builder.startObject("message"); - { - builder.field("type", "text"); - } - builder.endObject(); + builder.field("type", "text"); } builder.endObject(); } builder.endObject(); } builder.endObject(); - request.mapping("_doc", builder); // <1> + request.mapping(builder); // <1> //end::put-template-request-mappings-xcontent assertTrue(client.indices().putTemplate(request, RequestOptions.DEFAULT).isAcknowledged()); } - { - //tag::put-template-request-mappings-shortcut - request.mapping("_doc", "message", "type=text"); // <1> - //end::put-template-request-mappings-shortcut - assertTrue(client.indices().putTemplate(request, RequestOptions.DEFAULT).isAcknowledged()); - } // tag::put-template-request-aliases request.alias(new Alias("twitter_alias").filter(QueryBuilders.termQuery("user", "kimchy"))); // <1> @@ -2177,11 +2167,9 @@ public void testPutTemplate() throws Exception { " \"number_of_shards\": 1\n" + " },\n" + " \"mappings\": {\n" + - " \"_doc\": {\n" + - " \"properties\": {\n" + - " \"message\": {\n" + - " \"type\": \"text\"\n" + - " }\n" + + " \"properties\": {\n" + + " \"message\": {\n" + + " \"type\": \"text\"\n" + " }\n" + " }\n" + " },\n" + @@ -2244,13 +2232,11 @@ public void testGetTemplates() throws Exception { PutIndexTemplateRequest putRequest = new PutIndexTemplateRequest("my-template"); putRequest.patterns(Arrays.asList("pattern-1", "log-*")); putRequest.settings(Settings.builder().put("index.number_of_shards", 3).put("index.number_of_replicas", 1)); - putRequest.mapping("_doc", + putRequest.mapping( "{\n" + - " \"_doc\": {\n" + - " \"properties\": {\n" + - " \"message\": {\n" + - " \"type\": \"text\"\n" + - " }\n" + + " \"properties\": {\n" + + " \"message\": {\n" + + " \"type\": \"text\"\n" + " }\n" + " }\n" + "}", XContentType.JSON); @@ -2269,7 +2255,7 @@ public void testGetTemplates() throws Exception { // end::get-templates-request-masterTimeout // tag::get-templates-execute - GetIndexTemplatesResponse getTemplatesResponse = client.indices().getTemplate(request, RequestOptions.DEFAULT); + GetIndexTemplatesResponse getTemplatesResponse = client.indices().getIndexTemplate(request, RequestOptions.DEFAULT); // end::get-templates-execute // tag::get-templates-response @@ -2299,7 +2285,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::get-templates-execute-async - client.indices().getTemplateAsync(request, RequestOptions.DEFAULT, listener); // <1> + client.indices().getIndexTemplateAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::get-templates-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexTemplatesResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexTemplatesResponseTests.java new file mode 100644 index 0000000000000..d2f0c3d7eba88 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexTemplatesResponseTests.java @@ -0,0 +1,136 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.indices; + +import org.elasticsearch.cluster.metadata.AliasMetaData; +import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester; + +public class GetIndexTemplatesResponseTests extends ESTestCase { + + static final String mappingString = "{\"properties\":{" + + "\"f1\": {\"type\":\"text\"}," + + "\"f2\": {\"type\":\"keyword\"}" + + "}}"; + + + public void testFromXContent() throws IOException { + xContentTester(this::createParser, GetIndexTemplatesResponseTests::createTestInstance, GetIndexTemplatesResponseTests::toXContent, + GetIndexTemplatesResponse::fromXContent).supportsUnknownFields(false) + .assertEqualsConsumer(GetIndexTemplatesResponseTests::assertEqualInstances) + .shuffleFieldsExceptions(new String[] {"aliases", "mappings", "patterns", "settings"}) + .test(); + } + + private static void assertEqualInstances(GetIndexTemplatesResponse expectedInstance, GetIndexTemplatesResponse newInstance) { + assertEquals(expectedInstance, newInstance); + // Check there's no doc types at the root of the mapping + Map expectedMap = XContentHelper.convertToMap( + new BytesArray(mappingString), true, XContentType.JSON).v2(); + for (IndexTemplateMetaData template : newInstance.getIndexTemplates()) { + MappingMetaData mappingMD = template.mappings(); + if(mappingMD!=null) { + Map mappingAsMap = mappingMD.sourceAsMap(); + assertEquals(expectedMap, mappingAsMap); + } + } + } + + static GetIndexTemplatesResponse createTestInstance() { + List templates = new ArrayList<>(); + int numTemplates = between(0, 10); + for (int t = 0; t < numTemplates; t++) { + IndexTemplateMetaData.Builder templateBuilder = IndexTemplateMetaData.builder("template-" + t); + templateBuilder.patterns(IntStream.range(0, between(1, 5)).mapToObj(i -> "pattern-" + i).collect(Collectors.toList())); + int numAlias = between(0, 5); + for (int i = 0; i < numAlias; i++) { + templateBuilder.putAlias(AliasMetaData.builder(randomAlphaOfLengthBetween(1, 10))); + } + if (randomBoolean()) { + templateBuilder.settings(Settings.builder().put("index.setting-1", randomLong())); + } + if (randomBoolean()) { + templateBuilder.order(randomInt()); + } + if (randomBoolean()) { + templateBuilder.version(between(0, 100)); + } + if (randomBoolean()) { + try { + Map map = XContentHelper.convertToMap(new BytesArray(mappingString), true, XContentType.JSON).v2(); + MappingMetaData mapping = new MappingMetaData(MapperService.SINGLE_MAPPING_NAME, map); + templateBuilder.mapping(mapping); + } catch (IOException ex) { + throw new UncheckedIOException(ex); + } + } + templates.add(templateBuilder.build()); + } + return new GetIndexTemplatesResponse(templates); + } + + // As the client class GetIndexTemplatesResponse doesn't have toXContent method, adding this method here only for the test + static void toXContent(GetIndexTemplatesResponse response, XContentBuilder builder) throws IOException { + + //Create a server-side counterpart for the client-side class and call toXContent on it + + List serverIndexTemplates = new ArrayList<>(); + List clientIndexTemplates = response.getIndexTemplates(); + for (IndexTemplateMetaData clientITMD : clientIndexTemplates) { + org.elasticsearch.cluster.metadata.IndexTemplateMetaData.Builder serverTemplateBuilder = + org.elasticsearch.cluster.metadata.IndexTemplateMetaData.builder(clientITMD.name()); + + serverTemplateBuilder.patterns(clientITMD.patterns()); + + Iterator aliases = clientITMD.aliases().valuesIt(); + aliases.forEachRemaining((a)->serverTemplateBuilder.putAlias(a)); + + serverTemplateBuilder.settings(clientITMD.settings()); + serverTemplateBuilder.order(clientITMD.order()); + serverTemplateBuilder.version(clientITMD.version()); + if (clientITMD.mappings() != null) { + serverTemplateBuilder.putMapping(MapperService.SINGLE_MAPPING_NAME, clientITMD.mappings().source()); + } + serverIndexTemplates.add(serverTemplateBuilder.build()); + + } + org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse serverResponse = new + org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse(serverIndexTemplates); + serverResponse.toXContent(builder, ToXContent.EMPTY_PARAMS); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/PutIndexTemplateRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/PutIndexTemplateRequestTests.java new file mode 100644 index 0000000000000..8aab973982fc0 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/PutIndexTemplateRequestTests.java @@ -0,0 +1,116 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.indices; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Arrays; +import java.util.Collections; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.core.Is.is; + +public class PutIndexTemplateRequestTests extends AbstractXContentTestCase { + public void testValidateErrorMessage() throws Exception { + expectThrows(IllegalArgumentException.class, () -> new PutIndexTemplateRequest(null)); + expectThrows(IllegalArgumentException.class, () -> new PutIndexTemplateRequest("test").name(null)); + PutIndexTemplateRequest request = new PutIndexTemplateRequest("test"); + ActionRequestValidationException withoutPattern = request.validate(); + assertThat(withoutPattern.getMessage(), containsString("index patterns are missing")); + + request.name("foo"); + ActionRequestValidationException withoutIndexPatterns = request.validate(); + assertThat(withoutIndexPatterns.validationErrors(), hasSize(1)); + assertThat(withoutIndexPatterns.getMessage(), containsString("index patterns are missing")); + + request.patterns(Collections.singletonList("test-*")); + ActionRequestValidationException noError = request.validate(); + assertThat(noError, is(nullValue())); + } + + @Override + protected PutIndexTemplateRequest createTestInstance() { + PutIndexTemplateRequest request = new PutIndexTemplateRequest("test"); + if (randomBoolean()) { + request.version(randomInt()); + } + if (randomBoolean()) { + request.order(randomInt()); + } + request.patterns(Arrays.asList(generateRandomStringArray(20, 100, false, false))); + int numAlias = between(0, 5); + for (int i = 0; i < numAlias; i++) { + // some ASCII or Latin-1 control characters, especially newline, can lead to + // problems with yaml parsers, that's why we filter them here (see #30911) + Alias alias = new Alias(randomRealisticUnicodeOfLengthBetween(1, 10).replaceAll("\\p{Cc}", "")); + if (randomBoolean()) { + alias.indexRouting(randomRealisticUnicodeOfLengthBetween(1, 10)); + } + if (randomBoolean()) { + alias.searchRouting(randomRealisticUnicodeOfLengthBetween(1, 10)); + } + request.alias(alias); + } + if (randomBoolean()) { + try { + request.mapping(XContentFactory.jsonBuilder().startObject() + .startObject("properties") + .startObject("field-" + randomInt()).field("type", randomFrom("keyword", "text")).endObject() + .endObject().endObject()); + } catch (IOException ex) { + throw new UncheckedIOException(ex); + } + } + if (randomBoolean()) { + request.settings(Settings.builder().put("setting1", randomLong()).put("setting2", randomTimeValue()).build()); + } + return request; + } + + @Override + protected PutIndexTemplateRequest doParseInstance(XContentParser parser) throws IOException { + return new PutIndexTemplateRequest("test").source(parser.map()); + } + + @Override + protected void assertEqualInstances(PutIndexTemplateRequest expected, PutIndexTemplateRequest actual) { + assertNotSame(expected, actual); + assertThat(actual.version(), equalTo(expected.version())); + assertThat(actual.order(), equalTo(expected.order())); + assertThat(actual.patterns(), equalTo(expected.patterns())); + assertThat(actual.aliases(), equalTo(expected.aliases())); + assertThat(actual.mappings(), equalTo(expected.mappings())); + assertThat(actual.settings(), equalTo(expected.settings())); + } + + @Override + protected boolean supportsUnknownFields() { + return false; + } +} diff --git a/docs/java-rest/high-level/indices/put_template.asciidoc b/docs/java-rest/high-level/indices/put_template.asciidoc index 7618fc8ce7af5..3e3954308736d 100644 --- a/docs/java-rest/high-level/indices/put_template.asciidoc +++ b/docs/java-rest/high-level/indices/put_template.asciidoc @@ -39,8 +39,7 @@ template's patterns. -------------------------------------------------- include-tagged::{doc-tests-file}[{api}-request-mappings-json] -------------------------------------------------- -<1> The type to define -<2> The mapping for this type, provided as a JSON string +<1> The mapping, provided as a JSON string The mapping source can be provided in different ways in addition to the `String` example shown above: @@ -59,13 +58,6 @@ include-tagged::{doc-tests-file}[{api}-request-mappings-xcontent] <1> Mapping source provided as an `XContentBuilder` object, the Elasticsearch built-in helpers to generate JSON content -["source","java",subs="attributes,callouts,macros"] --------------------------------------------------- -include-tagged::{doc-tests-file}[{api}-request-mappings-shortcut] --------------------------------------------------- -<1> Mapping source provided as `Object` key-pairs, which gets converted to -JSON format - ==== Aliases The aliases of the template will define aliasing to the index whose name matches the template's patterns. A placeholder `{index}` can be used in an alias of a template. diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 43bb7401dba58..1b2503ccb99d5 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -35,6 +35,8 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.rest.action.admin.indices.RestGetIndexTemplateAction; +import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction; import org.elasticsearch.rest.action.document.RestBulkAction; import org.elasticsearch.rest.action.document.RestGetAction; import org.elasticsearch.rest.action.document.RestUpdateAction; @@ -921,6 +923,7 @@ public void testSnapshotRestore() throws IOException { // We therefore use the deprecated typed APIs when running against the current version. if (isRunningAgainstOldCluster() == false) { createTemplateRequest.addParameter(INCLUDE_TYPE_NAME_PARAMETER, "true"); + createTemplateRequest.setOptions(expectWarnings(RestPutIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)); } client().performRequest(createTemplateRequest); @@ -1122,6 +1125,7 @@ private void checkSnapshot(String snapshotName, int count, Version tookOnVersion // We therefore use the deprecated typed APIs when running against the current version. if (isRunningAgainstOldCluster() == false) { getTemplateRequest.addParameter(INCLUDE_TYPE_NAME_PARAMETER, "true"); + getTemplateRequest.setOptions(expectWarnings(RestGetIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)); } Map getTemplateResponse = entityAsMap(client().performRequest(getTemplateRequest)); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java index 9749aaa05b1a4..e5cd947f0f046 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java @@ -43,7 +43,7 @@ public class GetIndexTemplatesResponse extends ActionResponse implements ToXCont indexTemplates = new ArrayList<>(); } - GetIndexTemplatesResponse(List indexTemplates) { + public GetIndexTemplatesResponse(List indexTemplates) { this.indexTemplates = indexTemplates; } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java index 50370797aa6f2..707378eec4cf6 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java @@ -19,10 +19,12 @@ package org.elasticsearch.rest.action.admin.indices; +import org.apache.logging.log4j.LogManager; import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequest; import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.rest.BaseRestHandler; @@ -47,6 +49,10 @@ public class RestGetIndexTemplateAction extends BaseRestHandler { private static final Set RESPONSE_PARAMETERS = Collections.unmodifiableSet(Sets.union( Collections.singleton(INCLUDE_TYPE_NAME_PARAMETER), Settings.FORMAT_PARAMS)); + private static final DeprecationLogger deprecationLogger = new DeprecationLogger( + LogManager.getLogger(RestGetIndexTemplateAction.class)); + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal]" + + " Specifying include_type_name in get index template requests is deprecated."; public RestGetIndexTemplateAction(final Settings settings, final RestController controller) { super(settings); @@ -65,6 +71,9 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC final String[] names = Strings.splitStringByCommaToArray(request.param("name")); final GetIndexTemplatesRequest getIndexTemplatesRequest = new GetIndexTemplatesRequest(names); + if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { + deprecationLogger.deprecatedAndMaybeLog("get_index_template_include_type_name", TYPES_DEPRECATION_MESSAGE); + } getIndexTemplatesRequest.local(request.paramAsBoolean("local", getIndexTemplatesRequest.local())); getIndexTemplatesRequest.masterNodeTimeout(request.paramAsTime("master_timeout", getIndexTemplatesRequest.masterNodeTimeout())); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java index 2b72a724a8906..445d393fef82b 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java @@ -42,6 +42,9 @@ public class RestPutIndexTemplateAction extends BaseRestHandler { private static final DeprecationLogger deprecationLogger = new DeprecationLogger( LogManager.getLogger(RestPutIndexTemplateAction.class)); + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal]" + + " Specifying include_type_name in put index template requests is deprecated."+ + " The parameter will be removed in the next major version."; public RestPutIndexTemplateAction(Settings settings, RestController controller) { super(settings); @@ -57,6 +60,9 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { PutIndexTemplateRequest putRequest = new PutIndexTemplateRequest(request.param("name")); + if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { + deprecationLogger.deprecatedAndMaybeLog("put_index_template_with_types", TYPES_DEPRECATION_MESSAGE); + } if (request.hasParam("template")) { deprecationLogger.deprecated("Deprecated parameter[template] used, replaced by [index_patterns]"); putRequest.patterns(Collections.singletonList(request.param("template"))); diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java index ac0eb8f0d81a6..d1900aaee84d5 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.rest.action.admin.indices; +import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -31,8 +32,12 @@ import org.junit.Before; import java.io.IOException; +import java.util.HashMap; import java.util.Map; +import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER; +import static org.mockito.Mockito.mock; + public class RestPutIndexTemplateActionTests extends RestActionTestCase { private RestPutIndexTemplateAction action; @@ -45,7 +50,8 @@ public void testPrepareTypelessRequest() throws IOException { XContentBuilder content = XContentFactory.jsonBuilder().startObject() .startObject("mappings") .startObject("properties") - .startObject("field").field("type", "keyword").endObject() + .startObject("field1").field("type", "keyword").endObject() + .startObject("field2").field("type", "text").endObject() .endObject() .endObject() .startObject("aliases") @@ -58,6 +64,11 @@ public void testPrepareTypelessRequest() throws IOException { .withPath("/_template/_some_template") .withContent(BytesReference.bytes(content), XContentType.JSON) .build(); + action.prepareRequest(request, mock(NodeClient.class)); + + // Internally the above prepareRequest method calls prepareRequestSource to inject a + // default type into the mapping. Here we test that this does what is expected by + // explicitly calling that same helper function boolean includeTypeName = false; Map source = action.prepareRequestSource(request, includeTypeName); @@ -65,7 +76,8 @@ public void testPrepareTypelessRequest() throws IOException { .startObject("mappings") .startObject("_doc") .startObject("properties") - .startObject("field").field("type", "keyword").endObject() + .startObject("field1").field("type", "keyword").endObject() + .startObject("field2").field("type", "text").endObject() .endObject() .endObject() .endObject() @@ -78,4 +90,51 @@ public void testPrepareTypelessRequest() throws IOException { assertEquals(expectedContentAsMap, source); } + + public void testIncludeTypeName() throws IOException { + XContentBuilder typedContent = XContentFactory.jsonBuilder().startObject() + .startObject("mappings") + .startObject("my_doc") + .startObject("properties") + .startObject("field1").field("type", "keyword").endObject() + .startObject("field2").field("type", "text").endObject() + .endObject() + .endObject() + .endObject() + .startObject("aliases") + .startObject("read_alias").endObject() + .endObject() + .endObject(); + + Map params = new HashMap<>(); + params.put(INCLUDE_TYPE_NAME_PARAMETER, "true"); + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()) + .withMethod(RestRequest.Method.PUT) + .withParams(params) + .withPath("/_template/_some_template") + .withContent(BytesReference.bytes(typedContent), XContentType.JSON) + .build(); + action.prepareRequest(request, mock(NodeClient.class)); + assertWarnings(RestPutIndexTemplateAction.TYPES_DEPRECATION_MESSAGE); + boolean includeTypeName = true; + Map source = action.prepareRequestSource(request, includeTypeName); + + XContentBuilder expectedContent = XContentFactory.jsonBuilder().startObject() + .startObject("mappings") + .startObject("my_doc") + .startObject("properties") + .startObject("field1").field("type", "keyword").endObject() + .startObject("field2").field("type", "text").endObject() + .endObject() + .endObject() + .endObject() + .startObject("aliases") + .startObject("read_alias").endObject() + .endObject() + .endObject(); + Map expectedContentAsMap = XContentHelper.convertToMap( + BytesReference.bytes(expectedContent), true, expectedContent.contentType()).v2(); + + assertEquals(expectedContentAsMap, source); + } } From 0470ee1fcc757296e3f7f1f168a3a98539c01f44 Mon Sep 17 00:00:00 2001 From: markharwood Date: Tue, 29 Jan 2019 21:24:08 +0000 Subject: [PATCH 030/100] Docs fix - missing callout --- .../client/documentation/IndicesClientDocumentationIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java index 83244b90b950f..d358655f2355a 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java @@ -2101,7 +2101,7 @@ public void testPutTemplate() throws Exception { " \"type\": \"text\"\n" + " }\n" + " }\n" + - "}", // <2> + "}", XContentType.JSON); // end::put-template-request-mappings-json assertTrue(client.indices().putTemplate(request, RequestOptions.DEFAULT).isAcknowledged()); From 55b916afc04e7867d6fc5070445c3551b8d5a364 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 29 Jan 2019 15:58:31 -0700 Subject: [PATCH 031/100] Ensure task metadata not null in follow test (#37993) This commit fixes a potential race in the IndexFollowingIT. Currently it is possible that we fetch the task metadata, it is null, and that throws a null pointer exception. Assertbusy does not catch null pointer exceptions. This commit assertions that the metadata is not null. --- .../test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java index dec671f6e6340..55fcb6ace89fd 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java @@ -983,6 +983,7 @@ private CheckedRunnable assertTask(final int numberOfPrimaryShards, f return () -> { final ClusterState clusterState = followerClient().admin().cluster().prepareState().get().getState(); final PersistentTasksCustomMetaData taskMetadata = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + assertNotNull(taskMetadata); ListTasksRequest listTasksRequest = new ListTasksRequest(); listTasksRequest.setDetailed(true); From f5b9b4d89c9e28dba020cd53c39b582fb01d95bd Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Wed, 30 Jan 2019 15:32:45 +1100 Subject: [PATCH 032/100] Add version 6.6.1 (#37975) --- server/src/main/java/org/elasticsearch/Version.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index b66630344a7ea..e520d714bb931 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -116,10 +116,10 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_6_5_3 = new Version(V_6_5_3_ID, org.apache.lucene.util.Version.LUCENE_7_5_0); public static final int V_6_5_4_ID = 6050499; public static final Version V_6_5_4 = new Version(V_6_5_4_ID, org.apache.lucene.util.Version.LUCENE_7_5_0); - public static final int V_6_5_5_ID = 6050599; - public static final Version V_6_5_5 = new Version(V_6_5_5_ID, org.apache.lucene.util.Version.LUCENE_7_5_0); public static final int V_6_6_0_ID = 6060099; public static final Version V_6_6_0 = new Version(V_6_6_0_ID, org.apache.lucene.util.Version.LUCENE_7_6_0); + public static final int V_6_6_1_ID = 6060199; + public static final Version V_6_6_1 = new Version(V_6_6_1_ID, org.apache.lucene.util.Version.LUCENE_7_6_0); public static final int V_6_7_0_ID = 6070099; public static final Version V_6_7_0 = new Version(V_6_7_0_ID, org.apache.lucene.util.Version.LUCENE_7_7_0); public static final int V_7_0_0_ID = 7000099; @@ -142,10 +142,10 @@ public static Version fromId(int id) { return V_7_0_0; case V_6_7_0_ID: return V_6_7_0; + case V_6_6_1_ID: + return V_6_6_1; case V_6_6_0_ID: return V_6_6_0; - case V_6_5_5_ID: - return V_6_5_5; case V_6_5_4_ID: return V_6_5_4; case V_6_5_3_ID: From 57823c484fddafb65accce261621a3b051e647db Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Wed, 30 Jan 2019 06:22:31 +0100 Subject: [PATCH 033/100] Streamline S3 Repository- and Client-Settings (#37393) * Make repository settings override static settings * Cache clients according to settings * Introduce custom implementations for the AWS credentials here to be able to use them as part of a hash key --- docs/plugins/repository-s3.asciidoc | 26 ++++ .../repositories/s3/S3BasicCredentials.java | 62 +++++++++ .../s3/S3BasicSessionCredentials.java | 57 ++++++++ .../repositories/s3/S3BlobStore.java | 14 +- .../repositories/s3/S3ClientSettings.java | 126 +++++++++++++----- .../repositories/s3/S3Repository.java | 50 +------ .../repositories/s3/S3Service.java | 104 ++++++++++----- .../s3/AwsS3ServiceImplTests.java | 6 +- .../s3/S3BlobStoreRepositoryTests.java | 2 - .../repositories/s3/S3BlobStoreTests.java | 9 +- .../s3/S3ClientSettingsTests.java | 32 ++++- .../repositories/s3/S3RepositoryTests.java | 6 +- .../20_repository_permanent_credentials.yml | 80 +++++++++++ 13 files changed, 444 insertions(+), 130 deletions(-) create mode 100644 plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BasicCredentials.java create mode 100644 plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BasicSessionCredentials.java diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc index b0b87dda792fe..e7ab83ca6e69b 100644 --- a/docs/plugins/repository-s3.asciidoc +++ b/docs/plugins/repository-s3.asciidoc @@ -221,6 +221,32 @@ The following settings are supported: currently supported by the plugin. For more information about the different classes, see http://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html[AWS Storage Classes Guide] +NOTE: The option of defining client settings in the repository settings as documented below is considered deprecated: + +In addition to the above settings, you may also specify all non-secure client settings in the repository settings. +In this case, the client settings found in the repository settings will be merged with those of the named client used by the repository. +Conflicts between client and repository settings are resolved by the repository settings taking precedence over client settings. + +For example: + +[source,js] +---- +PUT _snapshot/my_s3_repository +{ + "type": "s3", + "settings": { + "client": "my_client_name", + "bucket": "my_bucket_name", + "endpoint": "my.s3.endpoint" + } +} +---- +// CONSOLE +// TEST[skip:we don't have s3 set up while testing this] + +This sets up a repository that uses all client settings from the client `my_client_named` except for the `endpoint` that is overridden +to `my.s3.endpoint` by the repository settings. + [[repository-s3-permissions]] ===== Recommended S3 Permissions diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BasicCredentials.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BasicCredentials.java new file mode 100644 index 0000000000000..f6dfb692d1de2 --- /dev/null +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BasicCredentials.java @@ -0,0 +1,62 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.repositories.s3; + +import com.amazonaws.auth.AWSCredentials; + +import java.util.Objects; + +class S3BasicCredentials implements AWSCredentials { + + private final String accessKey; + + private final String secretKey; + + S3BasicCredentials(String accessKey, String secretKey) { + this.accessKey = accessKey; + this.secretKey = secretKey; + } + + @Override + public final String getAWSAccessKeyId() { + return accessKey; + } + + @Override + public final String getAWSSecretKey() { + return secretKey; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final S3BasicCredentials that = (S3BasicCredentials) o; + return accessKey.equals(that.accessKey) && secretKey.equals(that.secretKey); + } + + @Override + public int hashCode() { + return Objects.hash(accessKey, secretKey); + } +} diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BasicSessionCredentials.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BasicSessionCredentials.java new file mode 100644 index 0000000000000..4057403aa2c0f --- /dev/null +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BasicSessionCredentials.java @@ -0,0 +1,57 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.repositories.s3; + +import com.amazonaws.auth.AWSSessionCredentials; + +import java.util.Objects; + +final class S3BasicSessionCredentials extends S3BasicCredentials implements AWSSessionCredentials { + + private final String sessionToken; + + S3BasicSessionCredentials(String accessKey, String secretKey, String sessionToken) { + super(accessKey, secretKey); + this.sessionToken = sessionToken; + } + + @Override + public String getSessionToken() { + return sessionToken; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final S3BasicSessionCredentials that = (S3BasicSessionCredentials) o; + return sessionToken.equals(that.sessionToken) && + getAWSAccessKeyId().equals(that.getAWSAccessKeyId()) && + getAWSSecretKey().equals(that.getAWSSecretKey()); + } + + @Override + public int hashCode() { + return Objects.hash(sessionToken, getAWSAccessKeyId(), getAWSSecretKey()); + } +} diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java index 27f2b305abd32..d4df4094fcf92 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java @@ -25,6 +25,7 @@ import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.amazonaws.services.s3.model.StorageClass; +import org.elasticsearch.cluster.metadata.RepositoryMetaData; import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; @@ -39,8 +40,6 @@ class S3BlobStore implements BlobStore { private final S3Service service; - private final String clientName; - private final String bucket; private final ByteSizeValue bufferSize; @@ -51,15 +50,18 @@ class S3BlobStore implements BlobStore { private final StorageClass storageClass; - S3BlobStore(S3Service service, String clientName, String bucket, boolean serverSideEncryption, - ByteSizeValue bufferSize, String cannedACL, String storageClass) { + private final RepositoryMetaData repositoryMetaData; + + S3BlobStore(S3Service service, String bucket, boolean serverSideEncryption, + ByteSizeValue bufferSize, String cannedACL, String storageClass, + RepositoryMetaData repositoryMetaData) { this.service = service; - this.clientName = clientName; this.bucket = bucket; this.serverSideEncryption = serverSideEncryption; this.bufferSize = bufferSize; this.cannedACL = initCannedACL(cannedACL); this.storageClass = initStorageClass(storageClass); + this.repositoryMetaData = repositoryMetaData; } @Override @@ -68,7 +70,7 @@ public String toString() { } public AmazonS3Reference clientReference() { - return service.client(clientName); + return service.client(repositoryMetaData); } public String bucket() { diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java index 58fca161415a4..ea45fbaf93dd3 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java @@ -21,9 +21,6 @@ import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.auth.BasicSessionCredentials; import org.elasticsearch.cluster.metadata.RepositoryMetaData; import org.elasticsearch.common.settings.SecureSetting; import org.elasticsearch.common.settings.SecureString; @@ -36,6 +33,7 @@ import java.util.HashMap; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Set; /** @@ -46,6 +44,9 @@ final class S3ClientSettings { // prefix for s3 client settings private static final String PREFIX = "s3.client."; + /** Placeholder client name for normalizing client settings in the repository settings. */ + private static final String PLACEHOLDER_CLIENT = "placeholder"; + /** The access key (ie login id) for connecting to s3. */ static final Setting.AffixSetting ACCESS_KEY_SETTING = Setting.affixKeySetting(PREFIX, "access_key", key -> SecureSetting.secureString(key, null)); @@ -95,7 +96,7 @@ final class S3ClientSettings { key -> Setting.boolSetting(key, ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope)); /** Credentials to authenticate with s3. */ - final AWSCredentials credentials; + final S3BasicCredentials credentials; /** The s3 endpoint the client should talk to, or empty string to use the default. */ final String endpoint; @@ -126,7 +127,7 @@ final class S3ClientSettings { /** Whether the s3 client should use an exponential backoff retry policy. */ final boolean throttleRetries; - protected S3ClientSettings(AWSCredentials credentials, String endpoint, Protocol protocol, + private S3ClientSettings(S3BasicCredentials credentials, String endpoint, Protocol protocol, String proxyHost, int proxyPort, String proxyUsername, String proxyPassword, int readTimeoutMillis, int maxRetries, boolean throttleRetries) { this.credentials = credentials; @@ -141,6 +142,51 @@ protected S3ClientSettings(AWSCredentials credentials, String endpoint, Protocol this.throttleRetries = throttleRetries; } + /** + * Overrides the settings in this instance with settings found in repository metadata. + * + * @param metadata RepositoryMetaData + * @return S3ClientSettings + */ + S3ClientSettings refine(RepositoryMetaData metadata) { + final Settings repoSettings = metadata.settings(); + // Normalize settings to placeholder client settings prefix so that we can use the affix settings directly + final Settings normalizedSettings = + Settings.builder().put(repoSettings).normalizePrefix(PREFIX + PLACEHOLDER_CLIENT + '.').build(); + final String newEndpoint = getRepoSettingOrDefault(ENDPOINT_SETTING, normalizedSettings, endpoint); + + final Protocol newProtocol = getRepoSettingOrDefault(PROTOCOL_SETTING, normalizedSettings, protocol); + final String newProxyHost = getRepoSettingOrDefault(PROXY_HOST_SETTING, normalizedSettings, proxyHost); + final int newProxyPort = getRepoSettingOrDefault(PROXY_PORT_SETTING, normalizedSettings, proxyPort); + final int newReadTimeoutMillis = Math.toIntExact( + getRepoSettingOrDefault(READ_TIMEOUT_SETTING, normalizedSettings, TimeValue.timeValueMillis(readTimeoutMillis)).millis()); + final int newMaxRetries = getRepoSettingOrDefault(MAX_RETRIES_SETTING, normalizedSettings, maxRetries); + final boolean newThrottleRetries = getRepoSettingOrDefault(USE_THROTTLE_RETRIES_SETTING, normalizedSettings, throttleRetries); + final S3BasicCredentials newCredentials; + if (checkDeprecatedCredentials(repoSettings)) { + newCredentials = loadDeprecatedCredentials(repoSettings); + } else { + newCredentials = credentials; + } + if (Objects.equals(endpoint, newEndpoint) && protocol == newProtocol && Objects.equals(proxyHost, newProxyHost) + && proxyPort == newProxyPort && newReadTimeoutMillis == readTimeoutMillis && maxRetries == newMaxRetries + && newThrottleRetries == throttleRetries && Objects.equals(credentials, newCredentials)) { + return this; + } + return new S3ClientSettings( + newCredentials, + newEndpoint, + newProtocol, + newProxyHost, + newProxyPort, + proxyUsername, + proxyPassword, + newReadTimeoutMillis, + newMaxRetries, + newThrottleRetries + ); + } + /** * Load all client settings from the given settings. * @@ -175,24 +221,24 @@ static boolean checkDeprecatedCredentials(Settings repositorySettings) { } // backcompat for reading keys out of repository settings (clusterState) - static BasicAWSCredentials loadDeprecatedCredentials(Settings repositorySettings) { + private static S3BasicCredentials loadDeprecatedCredentials(Settings repositorySettings) { assert checkDeprecatedCredentials(repositorySettings); try (SecureString key = S3Repository.ACCESS_KEY_SETTING.get(repositorySettings); SecureString secret = S3Repository.SECRET_KEY_SETTING.get(repositorySettings)) { - return new BasicAWSCredentials(key.toString(), secret.toString()); + return new S3BasicCredentials(key.toString(), secret.toString()); } } - static AWSCredentials loadCredentials(Settings settings, String clientName) { + private static S3BasicCredentials loadCredentials(Settings settings, String clientName) { try (SecureString accessKey = getConfigValue(settings, clientName, ACCESS_KEY_SETTING); SecureString secretKey = getConfigValue(settings, clientName, SECRET_KEY_SETTING); SecureString sessionToken = getConfigValue(settings, clientName, SESSION_TOKEN_SETTING)) { if (accessKey.length() != 0) { if (secretKey.length() != 0) { if (sessionToken.length() != 0) { - return new BasicSessionCredentials(accessKey.toString(), secretKey.toString(), sessionToken.toString()); + return new S3BasicSessionCredentials(accessKey.toString(), secretKey.toString(), sessionToken.toString()); } else { - return new BasicAWSCredentials(accessKey.toString(), secretKey.toString()); + return new S3BasicCredentials(accessKey.toString(), secretKey.toString()); } } else { throw new IllegalArgumentException("Missing secret key for s3 client [" + clientName + "]"); @@ -212,34 +258,48 @@ static AWSCredentials loadCredentials(Settings settings, String clientName) { // pkg private for tests /** Parse settings for a single client. */ static S3ClientSettings getClientSettings(final Settings settings, final String clientName) { - final AWSCredentials credentials = S3ClientSettings.loadCredentials(settings, clientName); - return getClientSettings(settings, clientName, credentials); - } - - static S3ClientSettings getClientSettings(final Settings settings, final String clientName, final AWSCredentials credentials) { try (SecureString proxyUsername = getConfigValue(settings, clientName, PROXY_USERNAME_SETTING); SecureString proxyPassword = getConfigValue(settings, clientName, PROXY_PASSWORD_SETTING)) { return new S3ClientSettings( - credentials, - getConfigValue(settings, clientName, ENDPOINT_SETTING), - getConfigValue(settings, clientName, PROTOCOL_SETTING), - getConfigValue(settings, clientName, PROXY_HOST_SETTING), - getConfigValue(settings, clientName, PROXY_PORT_SETTING), - proxyUsername.toString(), - proxyPassword.toString(), - Math.toIntExact(getConfigValue(settings, clientName, READ_TIMEOUT_SETTING).millis()), - getConfigValue(settings, clientName, MAX_RETRIES_SETTING), - getConfigValue(settings, clientName, USE_THROTTLE_RETRIES_SETTING) + S3ClientSettings.loadCredentials(settings, clientName), + getConfigValue(settings, clientName, ENDPOINT_SETTING), + getConfigValue(settings, clientName, PROTOCOL_SETTING), + getConfigValue(settings, clientName, PROXY_HOST_SETTING), + getConfigValue(settings, clientName, PROXY_PORT_SETTING), + proxyUsername.toString(), + proxyPassword.toString(), + Math.toIntExact(getConfigValue(settings, clientName, READ_TIMEOUT_SETTING).millis()), + getConfigValue(settings, clientName, MAX_RETRIES_SETTING), + getConfigValue(settings, clientName, USE_THROTTLE_RETRIES_SETTING) ); } } - static S3ClientSettings getClientSettings(final RepositoryMetaData metadata, final AWSCredentials credentials) { - final Settings.Builder builder = Settings.builder(); - for (final String key : metadata.settings().keySet()) { - builder.put(PREFIX + "provided" + "." + key, metadata.settings().get(key)); + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; } - return getClientSettings(builder.build(), "provided", credentials); + final S3ClientSettings that = (S3ClientSettings) o; + return proxyPort == that.proxyPort && + readTimeoutMillis == that.readTimeoutMillis && + maxRetries == that.maxRetries && + throttleRetries == that.throttleRetries && + Objects.equals(credentials, that.credentials) && + Objects.equals(endpoint, that.endpoint) && + protocol == that.protocol && + Objects.equals(proxyHost, that.proxyHost) && + Objects.equals(proxyUsername, that.proxyUsername) && + Objects.equals(proxyPassword, that.proxyPassword); + } + + @Override + public int hashCode() { + return Objects.hash(credentials, endpoint, protocol, proxyHost, proxyPort, proxyUsername, proxyPassword, + readTimeoutMillis, maxRetries, throttleRetries); } private static T getConfigValue(Settings settings, String clientName, @@ -248,4 +308,10 @@ private static T getConfigValue(Settings settings, String clientName, return concreteSetting.get(settings); } + private static T getRepoSettingOrDefault(Setting.AffixSetting setting, Settings normalizedSettings, T defaultValue) { + if (setting.getConcreteSettingForNamespace(PLACEHOLDER_CLIENT).exists(normalizedSettings)) { + return getConfigValue(normalizedSettings, PLACEHOLDER_CLIENT, setting); + } + return defaultValue; + } } diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index e0e34e40f3cf8..b1d29d89a59c0 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -19,7 +19,6 @@ package org.elasticsearch.repositories.s3; -import com.amazonaws.auth.BasicAWSCredentials; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; import org.elasticsearch.cluster.metadata.RepositoryMetaData; @@ -148,8 +147,6 @@ class S3Repository extends BlobStoreRepository { */ static final Setting BASE_PATH_SETTING = Setting.simpleString("base_path"); - private final Settings settings; - private final S3Service service; private final String bucket; @@ -168,9 +165,7 @@ class S3Repository extends BlobStoreRepository { private final String cannedACL; - private final String clientName; - - private final AmazonS3Reference reference; + private final RepositoryMetaData repositoryMetaData; /** * Constructs an s3 backed repository @@ -180,9 +175,10 @@ class S3Repository extends BlobStoreRepository { final NamedXContentRegistry namedXContentRegistry, final S3Service service) { super(metadata, settings, namedXContentRegistry); - this.settings = settings; this.service = service; + this.repositoryMetaData = metadata; + // Parse and validate the user's S3 Storage Class setting this.bucket = BUCKET_SETTING.get(metadata.settings()); if (bucket == null) { @@ -211,24 +207,10 @@ class S3Repository extends BlobStoreRepository { this.storageClass = STORAGE_CLASS_SETTING.get(metadata.settings()); this.cannedACL = CANNED_ACL_SETTING.get(metadata.settings()); - this.clientName = CLIENT_NAME.get(metadata.settings()); - - if (CLIENT_NAME.exists(metadata.settings()) && S3ClientSettings.checkDeprecatedCredentials(metadata.settings())) { - logger.warn( - "ignoring use of named client [{}] for repository [{}] as insecure credentials were specified", - clientName, - metadata.name()); - } - if (S3ClientSettings.checkDeprecatedCredentials(metadata.settings())) { // provided repository settings deprecationLogger.deprecated("Using s3 access/secret key from repository settings. Instead " + "store these in named clients and the elasticsearch keystore for secure settings."); - final BasicAWSCredentials insecureCredentials = S3ClientSettings.loadDeprecatedCredentials(metadata.settings()); - final S3ClientSettings s3ClientSettings = S3ClientSettings.getClientSettings(metadata, insecureCredentials); - this.reference = new AmazonS3Reference(service.buildClient(s3ClientSettings)); - } else { - reference = null; } logger.debug( @@ -243,21 +225,7 @@ class S3Repository extends BlobStoreRepository { @Override protected S3BlobStore createBlobStore() { - if (reference != null) { - assert S3ClientSettings.checkDeprecatedCredentials(metadata.settings()) : metadata.name(); - return new S3BlobStore(service, clientName, bucket, serverSideEncryption, bufferSize, cannedACL, storageClass) { - @Override - public AmazonS3Reference clientReference() { - if (reference.tryIncRef()) { - return reference; - } else { - throw new IllegalStateException("S3 client is closed"); - } - } - }; - } else { - return new S3BlobStore(service, clientName, bucket, serverSideEncryption, bufferSize, cannedACL, storageClass); - } + return new S3BlobStore(service, bucket, serverSideEncryption, bufferSize, cannedACL, storageClass, repositoryMetaData); } // only use for testing @@ -286,14 +254,4 @@ protected boolean isCompress() { protected ByteSizeValue chunkSize() { return chunkSize; } - - @Override - protected void doClose() { - if (reference != null) { - assert S3ClientSettings.checkDeprecatedCredentials(metadata.settings()) : metadata.name(); - reference.decRef(); - } - super.doClose(); - } - } diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java index a5ee861d0c38b..b0c8a6198130a 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java @@ -22,18 +22,20 @@ import com.amazonaws.ClientConfiguration; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper; import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.http.IdleConnectionReaper; -import com.amazonaws.internal.StaticCredentialsProvider; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.internal.Constants; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.cluster.metadata.RepositoryMetaData; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.MapBuilder; +import org.elasticsearch.common.settings.Settings; import java.io.Closeable; import java.io.IOException; @@ -43,56 +45,96 @@ class S3Service implements Closeable { - private static final Logger logger = LogManager.getLogger(S3Service.class); - private volatile Map clientsCache = emptyMap(); - private volatile Map clientsSettings = emptyMap(); + private volatile Map clientsCache = emptyMap(); + + /** + * Client settings calculated from static configuration and settings in the keystore. + */ + private volatile Map staticClientSettings = MapBuilder.newMapBuilder() + .put("default", S3ClientSettings.getClientSettings(Settings.EMPTY, "default")).immutableMap(); + + /** + * Client settings derived from those in {@link #staticClientSettings} by combining them with settings + * in the {@link RepositoryMetaData}. + */ + private volatile Map> derivedClientSettings = emptyMap(); /** * Refreshes the settings for the AmazonS3 clients and clears the cache of * existing clients. New clients will be build using these new settings. Old * clients are usable until released. On release they will be destroyed instead - * to being returned to the cache. + * of being returned to the cache. */ - public synchronized Map refreshAndClearCache(Map clientsSettings) { + public synchronized void refreshAndClearCache(Map clientsSettings) { // shutdown all unused clients // others will shutdown on their respective release releaseCachedClients(); - final Map prevSettings = this.clientsSettings; - this.clientsSettings = MapBuilder.newMapBuilder(clientsSettings).immutableMap(); - assert this.clientsSettings.containsKey("default") : "always at least have 'default'"; - // clients are built lazily by {@link client(String)} - return prevSettings; + this.staticClientSettings = MapBuilder.newMapBuilder(clientsSettings).immutableMap(); + derivedClientSettings = emptyMap(); + assert this.staticClientSettings.containsKey("default") : "always at least have 'default'"; + // clients are built lazily by {@link client} } /** - * Attempts to retrieve a client by name from the cache. If the client does not - * exist it will be created. + * Attempts to retrieve a client by its repository metadata and settings from the cache. + * If the client does not exist it will be created. */ - public AmazonS3Reference client(String clientName) { - AmazonS3Reference clientReference = clientsCache.get(clientName); - if ((clientReference != null) && clientReference.tryIncRef()) { - return clientReference; - } - synchronized (this) { - clientReference = clientsCache.get(clientName); - if ((clientReference != null) && clientReference.tryIncRef()) { + public AmazonS3Reference client(RepositoryMetaData repositoryMetaData) { + final S3ClientSettings clientSettings = settings(repositoryMetaData); + { + final AmazonS3Reference clientReference = clientsCache.get(clientSettings); + if (clientReference != null && clientReference.tryIncRef()) { return clientReference; } - final S3ClientSettings clientSettings = clientsSettings.get(clientName); - if (clientSettings == null) { - throw new IllegalArgumentException("Unknown s3 client name [" + clientName + "]. Existing client configs: " - + Strings.collectionToDelimitedString(clientsSettings.keySet(), ",")); + } + synchronized (this) { + final AmazonS3Reference existing = clientsCache.get(clientSettings); + if (existing != null && existing.tryIncRef()) { + return existing; } - logger.debug("creating S3 client with client_name [{}], endpoint [{}]", clientName, clientSettings.endpoint); - clientReference = new AmazonS3Reference(buildClient(clientSettings)); + final AmazonS3Reference clientReference = new AmazonS3Reference(buildClient(clientSettings)); clientReference.incRef(); - clientsCache = MapBuilder.newMapBuilder(clientsCache).put(clientName, clientReference).immutableMap(); + clientsCache = MapBuilder.newMapBuilder(clientsCache).put(clientSettings, clientReference).immutableMap(); return clientReference; } } + /** + * Either fetches {@link S3ClientSettings} for a given {@link RepositoryMetaData} from cached settings or creates them + * by overriding static client settings from {@link #staticClientSettings} with settings found in the repository metadata. + * @param repositoryMetaData Repository Metadata + * @return S3ClientSettings + */ + private S3ClientSettings settings(RepositoryMetaData repositoryMetaData) { + final String clientName = S3Repository.CLIENT_NAME.get(repositoryMetaData.settings()); + final S3ClientSettings staticSettings = staticClientSettings.get(clientName); + if (staticSettings != null) { + { + final S3ClientSettings existing = derivedClientSettings.getOrDefault(staticSettings, emptyMap()).get(repositoryMetaData); + if (existing != null) { + return existing; + } + } + synchronized (this) { + final Map derivedSettings = + derivedClientSettings.getOrDefault(staticSettings, emptyMap()); + final S3ClientSettings existing = derivedSettings.get(repositoryMetaData); + if (existing != null) { + return existing; + } + final S3ClientSettings newSettings = staticSettings.refine(repositoryMetaData); + derivedClientSettings = MapBuilder.newMapBuilder(derivedClientSettings).put( + staticSettings, MapBuilder.newMapBuilder(derivedSettings).put(repositoryMetaData, newSettings).immutableMap() + ).immutableMap(); + return newSettings; + } + } + throw new IllegalArgumentException("Unknown s3 client name [" + clientName + "]. Existing client configs: " + + Strings.collectionToDelimitedString(staticClientSettings.keySet(), ",")); + } + // proxy for testing AmazonS3 buildClient(final S3ClientSettings clientSettings) { final AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard(); @@ -141,17 +183,17 @@ static ClientConfiguration buildConfiguration(S3ClientSettings clientSettings) { // pkg private for tests static AWSCredentialsProvider buildCredentials(Logger logger, S3ClientSettings clientSettings) { - final AWSCredentials credentials = clientSettings.credentials; + final S3BasicCredentials credentials = clientSettings.credentials; if (credentials == null) { logger.debug("Using instance profile credentials"); return new PrivilegedInstanceProfileCredentialsProvider(); } else { logger.debug("Using basic key/secret credentials"); - return new StaticCredentialsProvider(credentials); + return new AWSStaticCredentialsProvider(credentials); } } - protected synchronized void releaseCachedClients() { + private synchronized void releaseCachedClients() { // the clients will shutdown when they will not be used anymore for (final AmazonS3Reference clientReference : clientsCache.values()) { clientReference.decRef(); diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java index 0c14f44d8b613..2f5982b77ce62 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java @@ -22,7 +22,7 @@ import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.internal.StaticCredentialsProvider; +import com.amazonaws.auth.AWSStaticCredentialsProvider; import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.Settings; @@ -61,7 +61,7 @@ public void testAWSCredentialsFromKeystore() { final String clientName = clientNamePrefix + i; final S3ClientSettings someClientSettings = allClientsSettings.get(clientName); final AWSCredentialsProvider credentialsProvider = S3Service.buildCredentials(logger, someClientSettings); - assertThat(credentialsProvider, instanceOf(StaticCredentialsProvider.class)); + assertThat(credentialsProvider, instanceOf(AWSStaticCredentialsProvider.class)); assertThat(credentialsProvider.getCredentials().getAWSAccessKeyId(), is(clientName + "_aws_access_key")); assertThat(credentialsProvider.getCredentials().getAWSSecretKey(), is(clientName + "_aws_secret_key")); } @@ -83,7 +83,7 @@ public void testSetDefaultCredential() { // test default exists and is an Instance provider final S3ClientSettings defaultClientSettings = allClientsSettings.get("default"); final AWSCredentialsProvider defaultCredentialsProvider = S3Service.buildCredentials(logger, defaultClientSettings); - assertThat(defaultCredentialsProvider, instanceOf(StaticCredentialsProvider.class)); + assertThat(defaultCredentialsProvider, instanceOf(AWSStaticCredentialsProvider.class)); assertThat(defaultCredentialsProvider.getCredentials().getAWSAccessKeyId(), is(awsAccessKey)); assertThat(defaultCredentialsProvider.getCredentials().getAWSSecretKey(), is(awsSecretKey)); } diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java index b4c2f81a3f8b8..739452dc178c4 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java @@ -59,7 +59,6 @@ public class S3BlobStoreRepositoryTests extends ESBlobStoreRepositoryIntegTestCa private static final ConcurrentMap blobs = new ConcurrentHashMap<>(); private static String bucket; - private static String client; private static ByteSizeValue bufferSize; private static boolean serverSideEncryption; private static String cannedACL; @@ -68,7 +67,6 @@ public class S3BlobStoreRepositoryTests extends ESBlobStoreRepositoryIntegTestCa @BeforeClass public static void setUpRepositorySettings() { bucket = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT); - client = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT); bufferSize = new ByteSizeValue(randomIntBetween(5, 50), ByteSizeUnit.MB); serverSideEncryption = randomBoolean(); if (randomBoolean()) { diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreTests.java index a44ad706b2361..6640a0dc4d71c 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreTests.java @@ -22,8 +22,10 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.CannedAccessControlList; import com.amazonaws.services.s3.model.StorageClass; +import org.elasticsearch.cluster.metadata.RepositoryMetaData; import org.elasticsearch.common.blobstore.BlobStore; import org.elasticsearch.common.blobstore.BlobStoreException; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.repositories.ESBlobStoreTestCase; @@ -114,15 +116,14 @@ public static S3BlobStore randomMockS3BlobStore() { storageClass = randomValueOtherThan(StorageClass.Glacier, () -> randomFrom(StorageClass.values())).toString(); } - final String theClientName = randomAlphaOfLength(4); final AmazonS3 client = new MockAmazonS3(new ConcurrentHashMap<>(), bucket, serverSideEncryption, cannedACL, storageClass); final S3Service service = new S3Service() { @Override - public synchronized AmazonS3Reference client(String clientName) { - assert theClientName.equals(clientName); + public synchronized AmazonS3Reference client(RepositoryMetaData repositoryMetaData) { return new AmazonS3Reference(client); } }; - return new S3BlobStore(service, theClientName, bucket, serverSideEncryption, bufferSize, cannedACL, storageClass); + return new S3BlobStore(service, bucket, serverSideEncryption, bufferSize, cannedACL, storageClass, + new RepositoryMetaData(bucket, "s3", Settings.EMPTY)); } } diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java index e629f43f8a3d3..53740672df329 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java @@ -21,8 +21,7 @@ import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.auth.BasicSessionCredentials; +import org.elasticsearch.cluster.metadata.RepositoryMetaData; import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; @@ -103,7 +102,7 @@ public void testCredentialsTypeWithAccessKeyAndSecretKey() { secureSettings.setString("s3.client.default.secret_key", "secret_key"); final Map settings = S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()); final S3ClientSettings defaultSettings = settings.get("default"); - BasicAWSCredentials credentials = (BasicAWSCredentials) defaultSettings.credentials; + S3BasicCredentials credentials = defaultSettings.credentials; assertThat(credentials.getAWSAccessKeyId(), is("access_key")); assertThat(credentials.getAWSSecretKey(), is("secret_key")); } @@ -115,9 +114,34 @@ public void testCredentialsTypeWithAccessKeyAndSecretKeyAndSessionToken() { secureSettings.setString("s3.client.default.session_token", "session_token"); final Map settings = S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()); final S3ClientSettings defaultSettings = settings.get("default"); - BasicSessionCredentials credentials = (BasicSessionCredentials) defaultSettings.credentials; + S3BasicSessionCredentials credentials = (S3BasicSessionCredentials) defaultSettings.credentials; assertThat(credentials.getAWSAccessKeyId(), is("access_key")); assertThat(credentials.getAWSSecretKey(), is("secret_key")); assertThat(credentials.getSessionToken(), is("session_token")); } + + public void testRefineWithRepoSettings() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("s3.client.default.access_key", "access_key"); + secureSettings.setString("s3.client.default.secret_key", "secret_key"); + secureSettings.setString("s3.client.default.session_token", "session_token"); + final S3ClientSettings baseSettings = S3ClientSettings.load( + Settings.builder().setSecureSettings(secureSettings).build()).get("default"); + + { + final S3ClientSettings refinedSettings = baseSettings.refine(new RepositoryMetaData("name", "type", Settings.EMPTY)); + assertTrue(refinedSettings == baseSettings); + } + + { + final String endpoint = "some.host"; + final S3ClientSettings refinedSettings = baseSettings.refine(new RepositoryMetaData("name", "type", + Settings.builder().put("endpoint", endpoint).build())); + assertThat(refinedSettings.endpoint, is(endpoint)); + S3BasicSessionCredentials credentials = (S3BasicSessionCredentials) refinedSettings.credentials; + assertThat(credentials.getAWSAccessKeyId(), is("access_key")); + assertThat(credentials.getAWSSecretKey(), is("secret_key")); + assertThat(credentials.getSessionToken(), is("session_token")); + } + } } diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java index 60ff802c4fc02..36fa8b684bbb9 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java @@ -29,7 +29,6 @@ import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; -import java.util.Collections; import java.util.Map; import static org.hamcrest.Matchers.containsString; @@ -49,13 +48,12 @@ public void shutdown() { private static class DummyS3Service extends S3Service { @Override - public AmazonS3Reference client(String clientName) { + public AmazonS3Reference client(RepositoryMetaData repositoryMetaData) { return new AmazonS3Reference(new DummyS3Client()); } @Override - public Map refreshAndClearCache(Map clientsSettings) { - return Collections.emptyMap(); + public void refreshAndClearCache(Map clientsSettings) { } @Override diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml index 1c64b66be3ad2..97bb36163b11d 100644 --- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml +++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml @@ -29,6 +29,86 @@ setup: snapshot: snapshot-two ignore: 404 +--- +"Try to create repository with broken endpoint override and named client": + + # Register repository with broken endpoint setting + - do: + catch: /repository_verification_exception/ + snapshot.create_repository: + repository: repository_broken + body: + type: s3 + settings: + bucket: ${permanent_bucket} + client: integration_test_permanent + base_path: "${permanent_base_path}" + endpoint: 127.0.0.1:5 + canned_acl: private + storage_class: standard + + # Turn of verification to be able to create the repo with broken endpoint setting + - do: + snapshot.create_repository: + verify: false + repository: repository_broken + body: + type: s3 + settings: + bucket: ${permanent_bucket} + client: integration_test_permanent + base_path: "${permanent_base_path}" + endpoint: 127.0.0.1:5 + canned_acl: private + storage_class: standard + + # Index documents + - do: + bulk: + refresh: true + body: + - index: + _index: docs + _type: doc + _id: 1 + - snapshot: one + - index: + _index: docs + _type: doc + _id: 2 + - snapshot: one + - index: + _index: docs + _type: doc + _id: 3 + - snapshot: one + + - do: + count: + index: docs + + - match: {count: 3} + + # Creating snapshot with broken repo should fail + - do: + catch: /repository_exception/ + snapshot.create: + repository: repository_broken + snapshot: snapshot-one + wait_for_completion: true + + # Creating snapshot with existing working repository should work + - do: + snapshot.create: + repository: repository_permanent + snapshot: snapshot-one + wait_for_completion: true + + - match: { snapshot.snapshot: snapshot-one } + - match: { snapshot.state: SUCCESS } + - match: { snapshot.include_global_state: true } + - match: { snapshot.shards.failed: 0 } + --- "Snapshot and Restore with repository-s3 using permanent credentials": From 99129d7786229ab2f537784ba148740a3053eb70 Mon Sep 17 00:00:00 2001 From: Tim Vernum Date: Wed, 30 Jan 2019 17:51:11 +1100 Subject: [PATCH 034/100] Fix exit code for Security CLI tools (#37956) The certgen, certutil and saml-metadata tools did not correctly return their exit code to the calling shell. These commands now explicitly exit with the code that was returned from the main(args, terminal) method. --- .../packaging/test/ArchiveTestCase.java | 16 +++++++++------- .../security/cli/CertificateGenerateTool.java | 3 +-- .../xpack/security/cli/CertificateTool.java | 2 +- .../security/authc/saml/SamlMetadataCommand.java | 2 +- .../test/resources/packaging/tests/certgen.bash | 9 +++++++++ 5 files changed, 21 insertions(+), 11 deletions(-) diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java index 7f2299d1fc3ea..6261cd62a2b12 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java @@ -287,15 +287,17 @@ public void test90SecurityCliPackaging() { if (distribution().equals(Distribution.DEFAULT_LINUX) || distribution().equals(Distribution.DEFAULT_WINDOWS)) { assertTrue(Files.exists(installation.lib.resolve("tools").resolve("security-cli"))); - Platforms.onLinux(() -> { - final Result result = sh.run(bin.elasticsearchCertutil + " help"); + final Platforms.PlatformAction action = () -> { + Result result = sh.run(bin.elasticsearchCertutil + " --help"); assertThat(result.stdout, containsString("Simplifies certificate creation for use with the Elastic Stack")); - }); - Platforms.onWindows(() -> { - final Result result = sh.run(bin.elasticsearchCertutil + " help"); - assertThat(result.stdout, containsString("Simplifies certificate creation for use with the Elastic Stack")); - }); + // Ensure that the exit code from the java command is passed back up through the shell script + result = sh.runIgnoreExitCode(bin.elasticsearchCertutil + " invalid-command"); + assertThat(result.exitCode, is(64)); + assertThat(result.stdout, containsString("Unknown command [invalid-command]")); + }; + Platforms.onLinux(action); + Platforms.onWindows(action); } else if (distribution().equals(Distribution.OSS_LINUX) || distribution().equals(Distribution.OSS_WINDOWS)) { assertFalse(Files.exists(installation.lib.resolve("tools").resolve("security-cli"))); } diff --git a/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateGenerateTool.java b/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateGenerateTool.java index 809e4a6d30524..4b30224dcd481 100644 --- a/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateGenerateTool.java +++ b/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateGenerateTool.java @@ -38,7 +38,6 @@ import org.elasticsearch.xpack.core.ssl.PemUtils; import javax.security.auth.x500.X500Principal; - import java.io.IOException; import java.io.OutputStream; import java.io.OutputStreamWriter; @@ -154,7 +153,7 @@ private static class InputFileParser { } public static void main(String[] args) throws Exception { - new CertificateGenerateTool().main(args, Terminal.DEFAULT); + exit(new CertificateGenerateTool().main(args, Terminal.DEFAULT)); } @Override diff --git a/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateTool.java b/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateTool.java index a966cac9109d9..435305b8a6914 100644 --- a/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateTool.java +++ b/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateTool.java @@ -134,7 +134,7 @@ private static class CertificateToolParser { public static void main(String[] args) throws Exception { - new CertificateTool().main(args, Terminal.DEFAULT); + exit(new CertificateTool().main(args, Terminal.DEFAULT)); } CertificateTool() { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlMetadataCommand.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlMetadataCommand.java index 6fa59269ac7e6..a60b2204095a5 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlMetadataCommand.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlMetadataCommand.java @@ -90,7 +90,7 @@ public class SamlMetadataCommand extends EnvironmentAwareCommand { private KeyStoreWrapper keyStoreWrapper; public static void main(String[] args) throws Exception { - new SamlMetadataCommand().main(args, Terminal.DEFAULT); + exit(new SamlMetadataCommand().main(args, Terminal.DEFAULT)); } public SamlMetadataCommand() { diff --git a/x-pack/qa/vagrant/src/test/resources/packaging/tests/certgen.bash b/x-pack/qa/vagrant/src/test/resources/packaging/tests/certgen.bash index dd41b93ea6b28..83f967c39891b 100644 --- a/x-pack/qa/vagrant/src/test/resources/packaging/tests/certgen.bash +++ b/x-pack/qa/vagrant/src/test/resources/packaging/tests/certgen.bash @@ -417,3 +417,12 @@ DATA_SETTINGS echo "$testSearch" | grep '"_index":"books"' echo "$testSearch" | grep '"_id":"0"' } + +@test "[$GROUP] exit code on failure" { + run sudo -E -u $MASTER_USER "$MASTER_HOME/bin/elasticsearch-certgen" --not-a-valid-option + [ "$status" -ne 0 ] || { + echo "Expected elasticsearch-certgen tool exit code to be non-zero" + echo "$output" + false + } +} From f51bc00fcf18fa0073641df7cf1caff7e68a9104 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 30 Jan 2019 07:58:26 +0100 Subject: [PATCH 035/100] Added ccr to xpack usage infrastructure (#37256) * Added ccr to xpack usage infrastructure Closes #37221 --- docs/reference/rest-api/info.asciidoc | 5 + .../elasticsearch/xpack/ccr/XPackUsageIT.java | 85 +++++++++ .../xpack/ccr/ESCCRRestTestCase.java | 16 ++ .../java/org/elasticsearch/xpack/ccr/Ccr.java | 13 ++ .../xpack/ccr/CCRFeatureSetTests.java | 123 +++++++++++++ .../xpack/ccr/CCRFeatureSetUsageTests.java | 25 +++ .../xpack/core/XPackClientPlugin.java | 2 + .../elasticsearch/xpack/core/XPackField.java | 2 + .../xpack/core/ccr/CCRFeatureSet.java | 174 ++++++++++++++++++ 9 files changed, 445 insertions(+) create mode 100644 x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/XPackUsageIT.java create mode 100644 x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java create mode 100644 x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetUsageTests.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/CCRFeatureSet.java diff --git a/docs/reference/rest-api/info.asciidoc b/docs/reference/rest-api/info.asciidoc index f02919cb39660..382b4ab78ff77 100644 --- a/docs/reference/rest-api/info.asciidoc +++ b/docs/reference/rest-api/info.asciidoc @@ -63,6 +63,11 @@ Example response: "expiry_date_in_millis" : 1542665112332 }, "features" : { + "ccr" : { + "description" : "Cross Cluster Replication", + "available" : true, + "enabled" : true + }, "graph" : { "description" : "Graph Data Exploration for the Elastic Stack", "available" : true, diff --git a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/XPackUsageIT.java b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/XPackUsageIT.java new file mode 100644 index 0000000000000..84271ce0acaf1 --- /dev/null +++ b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/XPackUsageIT.java @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ccr; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; + +import java.io.IOException; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.nullValue; + +public class XPackUsageIT extends ESCCRRestTestCase { + + public void testXPackCcrUsage() throws Exception { + if ("follow".equals(targetCluster) == false) { + logger.info("skipping test, waiting for target cluster [follow]" ); + return; + } + + Map previousUsage = getCcrUsage(); + putAutoFollowPattern("my_pattern", "leader_cluster", "messages-*"); + + // This index should be auto followed: + createLeaderIndex("messages-20200101"); + // This index will be followed manually + createLeaderIndex("my_index"); + followIndex("my_index", "my_index"); + + int previousFollowerIndicesCount = (Integer) previousUsage.get("follower_indices_count"); + int previousAutoFollowPatternsCount = (Integer) previousUsage.get("auto_follow_patterns_count"); + assertBusy(() -> { + Map ccrUsage = getCcrUsage(); + assertThat(ccrUsage.get("follower_indices_count"), equalTo(previousFollowerIndicesCount + 2)); + assertThat(ccrUsage.get("auto_follow_patterns_count"), equalTo(previousAutoFollowPatternsCount + 1)); + assertThat((Integer) ccrUsage.get("last_follow_time_in_millis"), greaterThanOrEqualTo(0)); + }); + + deleteAutoFollowPattern("my_pattern"); + pauseFollow("messages-20200101"); + closeIndex("messages-20200101"); + unfollow("messages-20200101"); + + pauseFollow("my_index"); + closeIndex("my_index"); + unfollow("my_index"); + + assertBusy(() -> { + Map ccrUsage = getCcrUsage(); + assertThat(ccrUsage.get("follower_indices_count"), equalTo(previousFollowerIndicesCount)); + assertThat(ccrUsage.get("auto_follow_patterns_count"), equalTo(previousAutoFollowPatternsCount)); + if (previousFollowerIndicesCount == 0) { + assertThat(ccrUsage.get("last_follow_time_in_millis"), nullValue()); + } else { + assertThat((Integer) ccrUsage.get("last_follow_time_in_millis"), greaterThanOrEqualTo(0)); + } + }); + } + + private void createLeaderIndex(String indexName) throws IOException { + try (RestClient leaderClient = buildLeaderClient()) { + Settings settings = Settings.builder() + .put("index.soft_deletes.enabled", true) + .build(); + Request request = new Request("PUT", "/" + indexName); + request.setJsonEntity("{\"settings\": " + Strings.toString(settings) + "}"); + assertOK(leaderClient.performRequest(request)); + } + } + + private Map getCcrUsage() throws IOException { + Request request = new Request("GET", "/_xpack/usage"); + Map response = toMap(client().performRequest(request)); + logger.info("xpack usage response={}", response); + return (Map) response.get("ccr"); + } + +} diff --git a/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java b/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java index 6cdb6b37961ff..656328d5ead9e 100644 --- a/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java +++ b/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java @@ -87,6 +87,22 @@ protected static void pauseFollow(RestClient client, String followIndex) throws assertOK(client.performRequest(new Request("POST", "/" + followIndex + "/_ccr/pause_follow"))); } + protected static void putAutoFollowPattern(String patternName, String remoteCluster, String indexPattern) throws IOException { + Request putPatternRequest = new Request("PUT", "/_ccr/auto_follow/" + patternName); + putPatternRequest.setJsonEntity("{\"leader_index_patterns\": [\"" + indexPattern + "\"], \"remote_cluster\": \"" + + remoteCluster + "\"}"); + assertOK(client().performRequest(putPatternRequest)); + } + + protected static void deleteAutoFollowPattern(String patternName) throws IOException { + Request putPatternRequest = new Request("DELETE", "/_ccr/auto_follow/" + patternName); + assertOK(client().performRequest(putPatternRequest)); + } + + protected static void unfollow(String followIndex) throws IOException { + assertOK(client().performRequest(new Request("POST", "/" + followIndex + "/_ccr/unfollow"))); + } + protected static void verifyDocuments(final String index, final int expectedNumDocs, final String query) throws IOException { verifyDocuments(index, expectedNumDocs, query, adminClient()); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java index 1345faaa5bcb0..a7fa69e7abd39 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.inject.Module; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; @@ -81,6 +82,7 @@ import org.elasticsearch.xpack.ccr.rest.RestResumeFollowAction; import org.elasticsearch.xpack.ccr.rest.RestUnfollowAction; import org.elasticsearch.xpack.core.XPackPlugin; +import org.elasticsearch.xpack.core.ccr.CCRFeatureSet; import org.elasticsearch.xpack.core.ccr.ShardFollowNodeTaskStatus; import org.elasticsearch.xpack.core.ccr.action.CcrStatsAction; import org.elasticsearch.xpack.core.ccr.action.DeleteAutoFollowPatternAction; @@ -126,6 +128,7 @@ public class Ccr extends Plugin implements ActionPlugin, PersistentTaskPlugin, E private final SetOnce restoreSourceService = new SetOnce<>(); private final SetOnce ccrSettings = new SetOnce<>(); private Client client; + private final boolean transportClientMode; /** * Construct an instance of the CCR container with the specified settings. @@ -147,6 +150,7 @@ public Ccr(final Settings settings) { this.settings = settings; this.enabled = CCR_ENABLED_SETTING.get(settings); this.ccrLicenseChecker = Objects.requireNonNull(ccrLicenseChecker); + this.transportClientMode = XPackPlugin.transportClientMode(settings); } @Override @@ -314,6 +318,15 @@ public void onIndexModule(IndexModule indexModule) { } } + @Override + public Collection createGuiceModules() { + if (transportClientMode) { + return Collections.emptyList(); + } + + return Collections.singleton(b -> XPackPlugin.bindFeatureSet(b, CCRFeatureSet.class)); + } + protected XPackLicenseState getLicenseState() { return XPackPlugin.getSharedLicenseState(); } @Override diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java new file mode 100644 index 0000000000000..b95e8fc7c4008 --- /dev/null +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ccr; + +import org.elasticsearch.Version; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.XPackFeatureSet; +import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; +import org.elasticsearch.xpack.core.ccr.CCRFeatureSet; +import org.junit.Before; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class CCRFeatureSetTests extends ESTestCase { + + private XPackLicenseState licenseState; + private ClusterService clusterService; + + @Before + public void init() throws Exception { + licenseState = mock(XPackLicenseState.class); + clusterService = mock(ClusterService.class); + } + + public void testAvailable() { + CCRFeatureSet featureSet = new CCRFeatureSet(Settings.EMPTY, licenseState, clusterService); + + when(licenseState.isCcrAllowed()).thenReturn(false); + assertThat(featureSet.available(), equalTo(false)); + + when(licenseState.isCcrAllowed()).thenReturn(true); + assertThat(featureSet.available(), equalTo(true)); + + featureSet = new CCRFeatureSet(Settings.EMPTY, null, clusterService); + assertThat(featureSet.available(), equalTo(false)); + } + + public void testEnabled() { + Settings.Builder settings = Settings.builder().put("xpack.ccr.enabled", false); + CCRFeatureSet featureSet = new CCRFeatureSet(settings.build(), licenseState, clusterService); + assertThat(featureSet.enabled(), equalTo(false)); + + settings = Settings.builder().put("xpack.ccr.enabled", true); + featureSet = new CCRFeatureSet(settings.build(), licenseState, clusterService); + assertThat(featureSet.enabled(), equalTo(true)); + } + + public void testName() { + CCRFeatureSet featureSet = new CCRFeatureSet(Settings.EMPTY, licenseState, clusterService); + assertThat(featureSet.name(), equalTo("ccr")); + } + + public void testNativeCodeInfo() { + CCRFeatureSet featureSet = new CCRFeatureSet (Settings.EMPTY, licenseState, clusterService); + assertNull(featureSet.nativeCodeInfo()); + } + + public void testUsageStats() throws Exception { + MetaData.Builder metaData = MetaData.builder(); + + int numFollowerIndices = randomIntBetween(0, 32); + for (int i = 0; i < numFollowerIndices; i++) { + IndexMetaData.Builder followerIndex = IndexMetaData.builder("follow_index" + i) + .settings(settings(Version.CURRENT).put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true)) + .numberOfShards(1) + .numberOfReplicas(0) + .creationDate(i) + .putCustom(Ccr.CCR_CUSTOM_METADATA_KEY, new HashMap<>()); + metaData.put(followerIndex); + } + + // Add a regular index, to check that we do not take that one into account: + IndexMetaData.Builder regularIndex = IndexMetaData.builder("my_index") + .settings(settings(Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0) + .creationDate(numFollowerIndices); + metaData.put(regularIndex); + + int numAutoFollowPatterns = randomIntBetween(0, 32); + Map patterns = new HashMap<>(numAutoFollowPatterns); + for (int i = 0; i < numAutoFollowPatterns; i++) { + AutoFollowMetadata.AutoFollowPattern pattern = new AutoFollowMetadata.AutoFollowPattern("remote_cluser", + Collections.singletonList("logs" + i + "*"), null, null, null, null, null, null, null, null, null, null, null); + patterns.put("pattern" + i, pattern); + } + metaData.putCustom(AutoFollowMetadata.TYPE, new AutoFollowMetadata(patterns, Collections.emptyMap(), Collections.emptyMap())); + + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).metaData(metaData).build(); + Mockito.when(clusterService.state()).thenReturn(clusterState); + + PlainActionFuture future = new PlainActionFuture<>(); + CCRFeatureSet ccrFeatureSet = new CCRFeatureSet(Settings.EMPTY, licenseState, clusterService); + ccrFeatureSet.usage(future); + CCRFeatureSet.Usage ccrUsage = (CCRFeatureSet.Usage) future.get(); + assertThat(ccrUsage.enabled(), equalTo(ccrFeatureSet.enabled())); + assertThat(ccrUsage.available(), equalTo(ccrFeatureSet.available())); + + assertThat(ccrUsage.getNumberOfFollowerIndices(), equalTo(numFollowerIndices)); + assertThat(ccrUsage.getLastFollowTimeInMillis(), greaterThanOrEqualTo(0L)); + assertThat(ccrUsage.getNumberOfAutoFollowPatterns(), equalTo(numAutoFollowPatterns)); + } + +} diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetUsageTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetUsageTests.java new file mode 100644 index 0000000000000..69e41ffddab43 --- /dev/null +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetUsageTests.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.ccr; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.core.ccr.CCRFeatureSet; + +public class CCRFeatureSetUsageTests extends AbstractWireSerializingTestCase { + + @Override + protected CCRFeatureSet.Usage createTestInstance() { + return new CCRFeatureSet.Usage(randomBoolean(), randomBoolean(), randomIntBetween(0, Integer.MAX_VALUE), + randomIntBetween(0, Integer.MAX_VALUE), randomNonNegativeLong()); + } + + @Override + protected Writeable.Reader instanceReader() { + return CCRFeatureSet.Usage::new; + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index 1b6f128318a2a..4e3feb3c8a2fa 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -41,6 +41,7 @@ import org.elasticsearch.xpack.core.action.XPackUsageAction; import org.elasticsearch.xpack.core.beats.BeatsFeatureSetUsage; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; +import org.elasticsearch.xpack.core.ccr.CCRFeatureSet; import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction; import org.elasticsearch.xpack.core.graph.GraphFeatureSetUsage; import org.elasticsearch.xpack.core.graph.action.GraphExploreAction; @@ -412,6 +413,7 @@ public List getNamedWriteables() { new NamedWriteableRegistry.Entry(MetaData.Custom.class, AutoFollowMetadata.TYPE, AutoFollowMetadata::new), new NamedWriteableRegistry.Entry(NamedDiff.class, AutoFollowMetadata.TYPE, in -> AutoFollowMetadata.readDiffFrom(MetaData.Custom.class, AutoFollowMetadata.TYPE, in)), + new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.CCR, CCRFeatureSet.Usage::new), // ILM new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.INDEX_LIFECYCLE, IndexLifecycleFeatureSetUsage::new), diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java index 0e6888dd80d73..0c763032e22ca 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java @@ -33,6 +33,8 @@ public final class XPackField { public static final String ROLLUP = "rollup"; /** Name constant for the index lifecycle feature. */ public static final String INDEX_LIFECYCLE = "ilm"; + /** Name constant for the CCR feature. */ + public static final String CCR = "ccr"; private XPackField() {} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/CCRFeatureSet.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/CCRFeatureSet.java new file mode 100644 index 0000000000000..f6e9e76d448f7 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/CCRFeatureSet.java @@ -0,0 +1,174 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ccr; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.xpack.core.XPackFeatureSet; +import org.elasticsearch.xpack.core.XPackField; +import org.elasticsearch.xpack.core.XPackSettings; + +import java.io.IOException; +import java.time.Instant; +import java.util.Map; +import java.util.Objects; + +public class CCRFeatureSet implements XPackFeatureSet { + + private final boolean enabled; + private final XPackLicenseState licenseState; + private final ClusterService clusterService; + + @Inject + public CCRFeatureSet(Settings settings, @Nullable XPackLicenseState licenseState, ClusterService clusterService) { + this.enabled = XPackSettings.CCR_ENABLED_SETTING.get(settings); + this.licenseState = licenseState; + this.clusterService = clusterService; + } + + @Override + public String name() { + return XPackField.CCR; + } + + @Override + public String description() { + return "Cross Cluster Replication"; + } + + @Override + public boolean available() { + return licenseState != null && licenseState.isCcrAllowed(); + } + + @Override + public boolean enabled() { + return enabled; + } + + @Override + public Map nativeCodeInfo() { + return null; + } + + @Override + public void usage(ActionListener listener) { + MetaData metaData = clusterService.state().metaData(); + + int numberOfFollowerIndices = 0; + long lastFollowerIndexCreationDate = 0L; + for (IndexMetaData imd : metaData) { + if (imd.getCustomData("ccr") != null) { + numberOfFollowerIndices++; + if (lastFollowerIndexCreationDate < imd.getCreationDate()) { + lastFollowerIndexCreationDate = imd.getCreationDate(); + } + } + } + AutoFollowMetadata autoFollowMetadata = metaData.custom(AutoFollowMetadata.TYPE); + int numberOfAutoFollowPatterns = autoFollowMetadata != null ? autoFollowMetadata.getPatterns().size() : 0; + + Long lastFollowTimeInMillis; + if (numberOfFollowerIndices == 0) { + // Otherwise we would return a value that makes no sense. + lastFollowTimeInMillis = null; + } else { + lastFollowTimeInMillis = Math.max(0, Instant.now().toEpochMilli() - lastFollowerIndexCreationDate); + } + + Usage usage = + new Usage(available(), enabled(), numberOfFollowerIndices, numberOfAutoFollowPatterns, lastFollowTimeInMillis); + listener.onResponse(usage); + } + + public static class Usage extends XPackFeatureSet.Usage { + + private final int numberOfFollowerIndices; + private final int numberOfAutoFollowPatterns; + private final Long lastFollowTimeInMillis; + + public Usage(boolean available, + boolean enabled, + int numberOfFollowerIndices, + int numberOfAutoFollowPatterns, + Long lastFollowTimeInMillis) { + super(XPackField.CCR, available, enabled); + this.numberOfFollowerIndices = numberOfFollowerIndices; + this.numberOfAutoFollowPatterns = numberOfAutoFollowPatterns; + this.lastFollowTimeInMillis = lastFollowTimeInMillis; + } + + public Usage(StreamInput in) throws IOException { + super(in); + numberOfFollowerIndices = in.readVInt(); + numberOfAutoFollowPatterns = in.readVInt(); + if (in.readBoolean()) { + lastFollowTimeInMillis = in.readVLong(); + } else { + lastFollowTimeInMillis = null; + } + } + + public int getNumberOfFollowerIndices() { + return numberOfFollowerIndices; + } + + public int getNumberOfAutoFollowPatterns() { + return numberOfAutoFollowPatterns; + } + + public Long getLastFollowTimeInMillis() { + return lastFollowTimeInMillis; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeVInt(numberOfFollowerIndices); + out.writeVInt(numberOfAutoFollowPatterns); + if (lastFollowTimeInMillis != null) { + out.writeBoolean(true); + out.writeVLong(lastFollowTimeInMillis); + } else { + out.writeBoolean(false); + } + } + + @Override + protected void innerXContent(XContentBuilder builder, Params params) throws IOException { + super.innerXContent(builder, params); + builder.field("follower_indices_count", numberOfFollowerIndices); + builder.field("auto_follow_patterns_count", numberOfAutoFollowPatterns); + if (lastFollowTimeInMillis != null) { + builder.field("last_follow_time_in_millis", lastFollowTimeInMillis); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Usage usage = (Usage) o; + return numberOfFollowerIndices == usage.numberOfFollowerIndices && + numberOfAutoFollowPatterns == usage.numberOfAutoFollowPatterns && + Objects.equals(lastFollowTimeInMillis, usage.lastFollowTimeInMillis); + } + + @Override + public int hashCode() { + return Objects.hash(numberOfFollowerIndices, numberOfAutoFollowPatterns, lastFollowTimeInMillis); + } + } +} From 5dcc805dc9fad2fde98610145639b5faa128ec11 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Wed, 30 Jan 2019 08:45:50 +0100 Subject: [PATCH 036/100] Restore a noop _all metadata field for 6x indices (#37808) This commit restores a noop version of the AllFieldMapper that is instanciated only for indices created in 6x. We need this metadata field mapper to be present in this version in order to allow the upgrade of indices that explicitly disable _all (enabled: false). The mapping of these indices contains a reference to the _all field that we cannot remove in 7 so we'll need to keep this metadata mapper in 7x. Since indices created in 6x will not be compatible with 8, we'll remove this noop mapper in the next major version. Closes #37429 --- .../test/mixed_cluster/10_basic.yml | 6 + .../test/old_cluster/10_basic.yml | 18 ++ .../test/upgraded_cluster/10_basic.yml | 14 ++ .../TransportGetFieldMappingsIndexAction.java | 4 +- ...TransportFieldCapabilitiesIndexAction.java | 3 +- .../index/mapper/AllFieldMapper.java | 175 ++++++++++++++++++ .../index/mapper/DocumentMapper.java | 5 +- .../index/mapper/DocumentMapperParser.java | 4 +- .../elasticsearch/indices/IndicesService.java | 5 +- .../indices/mapper/MapperRegistry.java | 18 +- .../index/mapper/AllFieldMapperTests.java | 68 +++++++ .../index/query/ExistsQueryBuilderTests.java | 3 - .../indices/IndicesModuleTests.java | 53 ++++-- .../indices/IndicesServiceTests.java | 6 +- 14 files changed, 354 insertions(+), 28 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml index f50a3fd9ea42d..2605836f8573c 100644 --- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml @@ -67,3 +67,9 @@ field3: value - match: { hits.total: 1 } - match: { hits.hits.0._id: q3 } + +--- +"Index with _all is available": + - do: + indices.get: + index: all-index diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml index ffa9e0ce2a6f8..9e06f767d4892 100644 --- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml @@ -203,3 +203,21 @@ tasks.get: wait_for_completion: true task_id: $task + +--- +"Create an index with _all explicitly disabled": + - skip: + features: warnings + - do: + warnings: + - "[_all] is deprecated in 6.0+ and will be removed in 7.0. As a replacement, you can use [copy_to] on mapping fields to create your own catch all field." + indices.create: + index: all-index + body: + mappings: + type: + _all: + enabled: false + properties: + field: + type: text diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml index 39c72dfd5334b..508a898e0cdb5 100644 --- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml @@ -125,3 +125,17 @@ wait_for_completion: true task_id: $task_id - match: { task.headers.X-Opaque-Id: "Reindexing Again" } + +--- +"Index with _all is available": + - do: + indices.get: + index: all-index + + - do: + indices.get_mapping: + index: all-index + + - is_true: all-index.mappings._all + - match: { all-index.mappings._all.enabled: false} + diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java index 67b3fe048bea1..c7415391675fc 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java @@ -20,6 +20,7 @@ package org.elasticsearch.action.admin.indices.mapping.get; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.single.shard.TransportSingleShardAction; @@ -91,7 +92,8 @@ protected ShardsIterator shards(ClusterState state, InternalRequest request) { protected GetFieldMappingsResponse shardOperation(final GetFieldMappingsIndexRequest request, ShardId shardId) { assert shardId != null; IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); - Predicate metadataFieldPredicate = indicesService::isMetaDataField; + Version indexCreatedVersion = indexService.mapperService().getIndexSettings().getIndexVersionCreated(); + Predicate metadataFieldPredicate = (f) -> indicesService.isMetaDataField(indexCreatedVersion, f); Predicate fieldPredicate = metadataFieldPredicate.or(indicesService.getFieldFilter().apply(shardId.getIndexName())); DocumentMapper mapper = indexService.mapperService().documentMapper(); diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java index c46933578f163..01c21544047ed 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java @@ -83,7 +83,8 @@ protected FieldCapabilitiesIndexResponse shardOperation(final FieldCapabilitiesI for (String field : fieldNames) { MappedFieldType ft = mapperService.fullName(field); if (ft != null) { - if (indicesService.isMetaDataField(field) || fieldPredicate.test(ft.name())) { + if (indicesService.isMetaDataField(mapperService.getIndexSettings().getIndexVersionCreated(), field) + || fieldPredicate.test(ft.name())) { FieldCapabilities fieldCap = new FieldCapabilities(field, ft.typeName(), ft.isSearchable(), ft.isAggregatable()); responseMap.put(field, fieldCap); } else { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java new file mode 100644 index 0000000000000..1427e67c5389b --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java @@ -0,0 +1,175 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.query.QueryShardContext; + +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +/** + * Noop mapper that ensures that mappings created in 6x that explicitly disable the _all field + * can be restored in this version. + * + * TODO: Remove in 8 + */ +public class AllFieldMapper extends MetadataFieldMapper { + public static final String NAME = "_all"; + public static final String CONTENT_TYPE = "_all"; + + public static class Defaults { + public static final MappedFieldType FIELD_TYPE = new AllFieldType(); + + static { + FIELD_TYPE.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); + FIELD_TYPE.setTokenized(true); + FIELD_TYPE.setName(NAME); + FIELD_TYPE.freeze(); + } + } + + public static class Builder extends MetadataFieldMapper.Builder { + private boolean disableExplicit = false; + + public Builder(MappedFieldType existing) { + super(NAME, existing == null ? Defaults.FIELD_TYPE : existing, Defaults.FIELD_TYPE); + builder = this; + } + + private Builder setDisableExplicit() { + this.disableExplicit = true; + return this; + } + + @Override + public AllFieldMapper build(BuilderContext context) { + return new AllFieldMapper(fieldType, context.indexSettings(), disableExplicit); + } + } + + public static class TypeParser implements MetadataFieldMapper.TypeParser { + @Override + public MetadataFieldMapper.Builder parse(String name, Map node, + ParserContext parserContext) throws MapperParsingException { + Builder builder = new Builder(parserContext.mapperService().fullName(NAME)); + for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { + Map.Entry entry = iterator.next(); + String fieldName = entry.getKey(); + if (fieldName.equals("enabled")) { + boolean enabled = XContentMapValues.nodeBooleanValue(entry.getValue(), "enabled"); + if (enabled) { + throw new IllegalArgumentException("[_all] is disabled in this version."); + } + builder.setDisableExplicit(); + iterator.remove(); + } + } + return builder; + } + + @Override + public MetadataFieldMapper getDefault(MappedFieldType fieldType, ParserContext context) { + final Settings indexSettings = context.mapperService().getIndexSettings().getSettings(); + return new AllFieldMapper(indexSettings, Defaults.FIELD_TYPE, false); + } + } + + static final class AllFieldType extends StringFieldType { + AllFieldType() { + } + + protected AllFieldType(AllFieldType ref) { + super(ref); + } + + @Override + public MappedFieldType clone() { + return new AllFieldType(this); + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + @Override + public Query existsQuery(QueryShardContext context) { + return new MatchNoDocsQuery(); + } + } + + private final boolean disableExplicit; + + private AllFieldMapper(Settings indexSettings, MappedFieldType existing, boolean disableExplicit) { + this(existing.clone(), indexSettings, disableExplicit); + } + + private AllFieldMapper(MappedFieldType fieldType, Settings indexSettings, boolean disableExplicit) { + super(NAME, fieldType, Defaults.FIELD_TYPE, indexSettings); + this.disableExplicit = disableExplicit; + } + + @Override + public void preParse(ParseContext context) throws IOException { + } + + @Override + public void postParse(ParseContext context) throws IOException { + super.parse(context); + } + + @Override + public void parse(ParseContext context) throws IOException { + // we parse in post parse + } + + @Override + protected void parseCreateField(ParseContext context, List fields) throws IOException { + // noop mapper + return; + } + + @Override + protected String contentType() { + return CONTENT_TYPE; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + boolean includeDefaults = params.paramAsBoolean("include_defaults", false); + if (includeDefaults || disableExplicit) { + builder.startObject(CONTENT_TYPE); + if (disableExplicit) { + builder.field("enabled", false); + } + builder.endObject(); + } + return builder; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index 058cf68e8e1f4..044e65c7ec6fb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -27,6 +27,7 @@ import org.apache.lucene.search.Weight; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchGenerationException; +import org.elasticsearch.Version; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; @@ -73,7 +74,9 @@ public Builder(RootObjectMapper.Builder builder, MapperService mapperService) { final String type = rootObjectMapper.name(); final DocumentMapper existingMapper = mapperService.documentMapper(type); - final Map metadataMapperParsers = mapperService.mapperRegistry.getMetadataMapperParsers(); + final Version indexCreatedVersion = mapperService.getIndexSettings().getIndexVersionCreated(); + final Map metadataMapperParsers = + mapperService.mapperRegistry.getMetadataMapperParsers(indexCreatedVersion); for (Map.Entry entry : metadataMapperParsers.entrySet()) { final String name = entry.getKey(); final MetadataFieldMapper existingMetadataMapper = existingMapper == null diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java index e388dd7ebcd00..db7954e9bd76a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java @@ -65,8 +65,8 @@ public DocumentMapperParser(IndexSettings indexSettings, MapperService mapperSer this.similarityService = similarityService; this.queryShardContextSupplier = queryShardContextSupplier; this.typeParsers = mapperRegistry.getMapperParsers(); - this.rootTypeParsers = mapperRegistry.getMetadataMapperParsers(); - indexVersionCreated = indexSettings.getIndexVersionCreated(); + this.indexVersionCreated = indexSettings.getIndexVersionCreated(); + this.rootTypeParsers = mapperRegistry.getMetadataMapperParsers(indexVersionCreated); } public Mapper.TypeParser.ParserContext parserContext(String type) { diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index fa42776403dca..cca63c015f1c7 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -29,6 +29,7 @@ import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.stats.CommonStats; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag; @@ -1382,8 +1383,8 @@ public Function> getFieldFilter() { /** * Returns true if the provided field is a registered metadata field (including ones registered via plugins), false otherwise. */ - public boolean isMetaDataField(String field) { - return mapperRegistry.isMetaDataField(field); + public boolean isMetaDataField(Version indexCreatedVersion, String field) { + return mapperRegistry.isMetaDataField(indexCreatedVersion, field); } /** diff --git a/server/src/main/java/org/elasticsearch/indices/mapper/MapperRegistry.java b/server/src/main/java/org/elasticsearch/indices/mapper/MapperRegistry.java index 41d563c2037ea..c79da36200e4f 100644 --- a/server/src/main/java/org/elasticsearch/indices/mapper/MapperRegistry.java +++ b/server/src/main/java/org/elasticsearch/indices/mapper/MapperRegistry.java @@ -19,6 +19,8 @@ package org.elasticsearch.indices.mapper; +import org.elasticsearch.Version; +import org.elasticsearch.index.mapper.AllFieldMapper; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.plugins.MapperPlugin; @@ -36,6 +38,7 @@ public final class MapperRegistry { private final Map mapperParsers; private final Map metadataMapperParsers; + private final Map metadataMapperParsers6x; private final Function> fieldFilter; @@ -43,6 +46,11 @@ public MapperRegistry(Map mapperParsers, Map metadataMapperParsers, Function> fieldFilter) { this.mapperParsers = Collections.unmodifiableMap(new LinkedHashMap<>(mapperParsers)); this.metadataMapperParsers = Collections.unmodifiableMap(new LinkedHashMap<>(metadataMapperParsers)); + // add the _all field mapper for indices created in 6x + Map metadata6x = new LinkedHashMap<>(); + metadata6x.put(AllFieldMapper.NAME, new AllFieldMapper.TypeParser()); + metadata6x.putAll(metadataMapperParsers); + this.metadataMapperParsers6x = Collections.unmodifiableMap(metadata6x); this.fieldFilter = fieldFilter; } @@ -58,15 +66,15 @@ public Map getMapperParsers() { * Return a map of the meta mappers that have been registered. The * returned map uses the name of the field as a key. */ - public Map getMetadataMapperParsers() { - return metadataMapperParsers; + public Map getMetadataMapperParsers(Version indexCreatedVersion) { + return indexCreatedVersion.onOrAfter(Version.V_7_0_0) ? metadataMapperParsers : metadataMapperParsers6x; } /** - * Returns true if the provide field is a registered metadata field, false otherwise + * Returns true if the provided field is a registered metadata field, false otherwise */ - public boolean isMetaDataField(String field) { - return getMetadataMapperParsers().containsKey(field); + public boolean isMetaDataField(Version indexCreatedVersion, String field) { + return getMetadataMapperParsers(indexCreatedVersion).containsKey(field); } /** diff --git a/server/src/test/java/org/elasticsearch/index/mapper/AllFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/AllFieldMapperTests.java index 62f3495ee172b..34200b51cb317 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/AllFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/AllFieldMapperTests.java @@ -19,6 +19,8 @@ package org.elasticsearch.index.mapper; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; @@ -26,9 +28,75 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.test.VersionUtils; + +import static org.hamcrest.CoreMatchers.containsString; public class AllFieldMapperTests extends ESSingleNodeTestCase { + @Override + protected boolean forbidPrivateIndexSettings() { + return false; + } + + public void testAllDisabled() throws Exception { + { + final Version version = VersionUtils.randomVersionBetween(random(), + Version.V_6_0_0, Version.V_7_0_0.minimumCompatibilityVersion()); + IndexService indexService = createIndex("test_6x", + Settings.builder() + .put(IndexMetaData.SETTING_VERSION_CREATED, version) + .build() + ); + String mappingDisabled = Strings.toString(XContentFactory.jsonBuilder().startObject() + .startObject("_all") + .field("enabled", false) + .endObject().endObject() + ); + indexService.mapperService().merge("_doc", new CompressedXContent(mappingDisabled), MergeReason.MAPPING_UPDATE); + + String mappingEnabled = Strings.toString(XContentFactory.jsonBuilder().startObject() + .startObject("_all") + .field("enabled", true) + .endObject().endObject() + ); + MapperParsingException exc = expectThrows(MapperParsingException.class, + () -> indexService.mapperService().merge("_doc", new CompressedXContent(mappingEnabled), MergeReason.MAPPING_UPDATE)); + assertThat(exc.getMessage(), containsString("[_all] is disabled in this version.")); + } + { + IndexService indexService = createIndex("test"); + String mappingEnabled = Strings.toString(XContentFactory.jsonBuilder().startObject() + .startObject("_all") + .field("enabled", true) + .endObject().endObject() + ); + MapperParsingException exc = expectThrows(MapperParsingException.class, + () -> indexService.mapperService().merge("_doc", new CompressedXContent(mappingEnabled), MergeReason.MAPPING_UPDATE)); + assertThat(exc.getMessage(), containsString("unsupported parameters: [_all")); + + String mappingDisabled = Strings.toString(XContentFactory.jsonBuilder().startObject() + .startObject("_all") + .field("enabled", false) + .endObject().endObject() + ); + exc = expectThrows(MapperParsingException.class, + () -> indexService.mapperService().merge("_doc", new CompressedXContent(mappingDisabled), MergeReason.MAPPING_UPDATE)); + assertThat(exc.getMessage(), containsString("unsupported parameters: [_all")); + + String mappingAll = Strings.toString(XContentFactory.jsonBuilder().startObject() + .startObject("_all").endObject().endObject() + ); + exc = expectThrows(MapperParsingException.class, + () -> indexService.mapperService().merge("_doc", new CompressedXContent(mappingAll), MergeReason.MAPPING_UPDATE)); + assertThat(exc.getMessage(), containsString("unsupported parameters: [_all")); + + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().endObject()); + indexService.mapperService().merge("_doc", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); + assertEquals("{\"_doc\":{}}", indexService.mapperService().documentMapper("_doc").mapping().toString()); + } + } + public void testUpdateDefaultSearchAnalyzer() throws Exception { IndexService indexService = createIndex("test", Settings.builder() .put("index.analysis.analyzer.default_search.type", "custom") diff --git a/server/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java index ad02209bf5dae..a5329856630d5 100644 --- a/server/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java @@ -28,7 +28,6 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.test.AbstractQueryTestCase; @@ -55,8 +54,6 @@ protected ExistsQueryBuilder doCreateTestQueryBuilder() { if (randomBoolean()) { if (randomBoolean()) { fieldPattern = fieldPattern + "*"; - } else { - fieldPattern = MetaData.ALL; } } return new ExistsQueryBuilder(fieldPattern); diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java index 9b88c6ab8f20c..f31ac0627138e 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java @@ -19,6 +19,8 @@ package org.elasticsearch.indices; +import org.elasticsearch.Version; +import org.elasticsearch.index.mapper.AllFieldMapper; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.IgnoredFieldMapper; @@ -36,6 +38,7 @@ import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.VersionUtils; import java.util.ArrayList; import java.util.Arrays; @@ -87,14 +90,36 @@ public Map getMetadataMappers() { RoutingFieldMapper.NAME, IndexFieldMapper.NAME, SourceFieldMapper.NAME, TypeFieldMapper.NAME, VersionFieldMapper.NAME, SeqNoFieldMapper.NAME, FieldNamesFieldMapper.NAME}; + private static String[] EXPECTED_METADATA_FIELDS_6x = new String[]{AllFieldMapper.NAME, IgnoredFieldMapper.NAME, + IdFieldMapper.NAME, RoutingFieldMapper.NAME, IndexFieldMapper.NAME, SourceFieldMapper.NAME, TypeFieldMapper.NAME, + VersionFieldMapper.NAME, SeqNoFieldMapper.NAME, FieldNamesFieldMapper.NAME}; + + public void testBuiltinMappers() { IndicesModule module = new IndicesModule(Collections.emptyList()); - assertFalse(module.getMapperRegistry().getMapperParsers().isEmpty()); - assertFalse(module.getMapperRegistry().getMetadataMapperParsers().isEmpty()); - Map metadataMapperParsers = module.getMapperRegistry().getMetadataMapperParsers(); - int i = 0; - for (String field : metadataMapperParsers.keySet()) { - assertEquals(EXPECTED_METADATA_FIELDS[i++], field); + { + Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, Version.V_7_0_0.minimumCompatibilityVersion()); + assertFalse(module.getMapperRegistry().getMapperParsers().isEmpty()); + assertFalse(module.getMapperRegistry().getMetadataMapperParsers(version).isEmpty()); + Map metadataMapperParsers = + module.getMapperRegistry().getMetadataMapperParsers(version); + assertEquals(EXPECTED_METADATA_FIELDS_6x.length, metadataMapperParsers.size()); + int i = 0; + for (String field : metadataMapperParsers.keySet()) { + assertEquals(EXPECTED_METADATA_FIELDS_6x[i++], field); + } + } + { + Version version = VersionUtils.randomVersionBetween(random(), Version.V_7_0_0, Version.CURRENT); + assertFalse(module.getMapperRegistry().getMapperParsers().isEmpty()); + assertFalse(module.getMapperRegistry().getMetadataMapperParsers(version).isEmpty()); + Map metadataMapperParsers = + module.getMapperRegistry().getMetadataMapperParsers(version); + assertEquals(EXPECTED_METADATA_FIELDS.length, metadataMapperParsers.size()); + int i = 0; + for (String field : metadataMapperParsers.keySet()) { + assertEquals(EXPECTED_METADATA_FIELDS[i++], field); + } } } @@ -102,11 +127,15 @@ public void testBuiltinWithPlugins() { IndicesModule noPluginsModule = new IndicesModule(Collections.emptyList()); IndicesModule module = new IndicesModule(fakePlugins); MapperRegistry registry = module.getMapperRegistry(); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, Version.V_7_0_0.minimumCompatibilityVersion()); assertThat(registry.getMapperParsers().size(), greaterThan(noPluginsModule.getMapperRegistry().getMapperParsers().size())); - assertThat(registry.getMetadataMapperParsers().size(), - greaterThan(noPluginsModule.getMapperRegistry().getMetadataMapperParsers().size())); - Map metadataMapperParsers = module.getMapperRegistry().getMetadataMapperParsers(); + assertThat(registry.getMetadataMapperParsers(version).size(), + greaterThan(noPluginsModule.getMapperRegistry().getMetadataMapperParsers(version).size())); + Map metadataMapperParsers = module.getMapperRegistry().getMetadataMapperParsers(version); Iterator iterator = metadataMapperParsers.keySet().iterator(); + if (version.before(Version.V_7_0_0)) { + assertEquals(AllFieldMapper.NAME, iterator.next()); + } assertEquals(IgnoredFieldMapper.NAME, iterator.next()); String last = null; while(iterator.hasNext()) { @@ -187,13 +216,15 @@ public Map getMetadataMappers() { public void testFieldNamesIsLast() { IndicesModule module = new IndicesModule(Collections.emptyList()); - List fieldNames = new ArrayList<>(module.getMapperRegistry().getMetadataMapperParsers().keySet()); + Version version = VersionUtils.randomCompatibleVersion(random(), Version.CURRENT); + List fieldNames = new ArrayList<>(module.getMapperRegistry().getMetadataMapperParsers(version).keySet()); assertEquals(FieldNamesFieldMapper.NAME, fieldNames.get(fieldNames.size() - 1)); } public void testFieldNamesIsLastWithPlugins() { IndicesModule module = new IndicesModule(fakePlugins); - List fieldNames = new ArrayList<>(module.getMapperRegistry().getMetadataMapperParsers().keySet()); + Version version = VersionUtils.randomCompatibleVersion(random(), Version.CURRENT); + List fieldNames = new ArrayList<>(module.getMapperRegistry().getMetadataMapperParsers(version).keySet()); assertEquals(FieldNamesFieldMapper.NAME, fieldNames.get(fieldNames.size() - 1)); } diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java index b68ec6d0598e5..60dbad99795f3 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java @@ -68,6 +68,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.IndexSettingsModule; +import org.elasticsearch.test.VersionUtils; import org.elasticsearch.test.hamcrest.RegexMatcher; import java.io.IOException; @@ -515,9 +516,10 @@ public void testStatsByShardDoesNotDieFromExpectedExceptions() { public void testIsMetaDataField() { IndicesService indicesService = getIndicesService(); - assertFalse(indicesService.isMetaDataField(randomAlphaOfLengthBetween(10, 15))); + final Version randVersion = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, Version.CURRENT); + assertFalse(indicesService.isMetaDataField(randVersion, randomAlphaOfLengthBetween(10, 15))); for (String builtIn : IndicesModule.getBuiltInMetaDataFields()) { - assertTrue(indicesService.isMetaDataField(builtIn)); + assertTrue(indicesService.isMetaDataField(randVersion, builtIn)); } } From 2732bb5cf3d854ab404a5ebe44a2f116cc3f1073 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Wed, 30 Jan 2019 08:46:14 +0100 Subject: [PATCH 037/100] Fix fetch source option in expand search phase (#37908) This change fixes the copy of the fetch source option into the expand search request that is used to retrieve the documents of each collapsed group. Closes #23829 --- .../org/elasticsearch/action/search/ExpandSearchPhase.java | 3 ++- .../elasticsearch/action/search/ExpandSearchPhaseTests.java | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java index 10a85b723166c..afc81b21da4d5 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java @@ -129,7 +129,8 @@ private SearchSourceBuilder buildExpandSearchSourceBuilder(InnerHitBuilder optio options.getSorts().forEach(groupSource::sort); } if (options.getFetchSourceContext() != null) { - if (options.getFetchSourceContext().includes() == null && options.getFetchSourceContext().excludes() == null) { + if (options.getFetchSourceContext().includes().length == 0 && + options.getFetchSourceContext().excludes().length == 0) { groupSource.fetchSource(options.getFetchSourceContext().fetchSource()); } else { groupSource.fetchSource(options.getFetchSourceContext().includes(), diff --git a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java index 328950e4f3569..d9de69a1c6c62 100644 --- a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java @@ -252,6 +252,9 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL assertTrue(request.requests().stream().allMatch((r) -> version == r.source().version())); assertTrue(request.requests().stream().allMatch((r) -> seqNoAndTerm == r.source().seqNoAndPrimaryTerm())); assertTrue(request.requests().stream().allMatch((r) -> postFilter.equals(r.source().postFilter()))); + assertTrue(request.requests().stream().allMatch((r) -> r.source().fetchSource().fetchSource() == false)); + assertTrue(request.requests().stream().allMatch((r) -> r.source().fetchSource().includes().length == 0)); + assertTrue(request.requests().stream().allMatch((r) -> r.source().fetchSource().excludes().length == 0)); } }; mockSearchPhaseContext.getRequest().source(new SearchSourceBuilder() @@ -259,6 +262,7 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL new CollapseBuilder("someField") .setInnerHits(new InnerHitBuilder().setName("foobarbaz").setVersion(version).setSeqNoAndPrimaryTerm(seqNoAndTerm)) ) + .fetchSource(false) .postFilter(QueryBuilders.existsQuery("foo"))) .preference("foobar") .routing("baz"); From 4da7a446489674beff665a3093037fdb25f93787 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 30 Jan 2019 09:37:41 +0100 Subject: [PATCH 038/100] Documented default values for index follow request parameters. (#37917) --- .../ccr/apis/follow-request-body.asciidoc | 62 ++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/docs/reference/ccr/apis/follow-request-body.asciidoc b/docs/reference/ccr/apis/follow-request-body.asciidoc index 7215cc01302a1..ec384deee8b81 100644 --- a/docs/reference/ccr/apis/follow-request-body.asciidoc +++ b/docs/reference/ccr/apis/follow-request-body.asciidoc @@ -1,3 +1,5 @@ +[role="xpack"] +[testenv="platinum"] `max_read_request_operation_count`:: (integer) the maximum number of operations to pull per read from the remote cluster @@ -41,4 +43,62 @@ remote cluster when the follower index is synchronized with the leader index; when the timeout has elapsed, the poll for operations will return to the follower so that it can update some statistics, and then the follower will - immediately attempt to read from the leader again \ No newline at end of file + immediately attempt to read from the leader again + +===== Default values + +////////////////////////// + +[source,js] +-------------------------------------------------- +PUT /follower_index/_ccr/follow +{ + "remote_cluster" : "remote_cluster", + "leader_index" : "leader_index" +} +-------------------------------------------------- +// CONSOLE +// TESTSETUP +// TEST[setup:remote_cluster_and_leader_index] + +[source,js] +-------------------------------------------------- +POST /follower_index/_ccr/pause_follow +-------------------------------------------------- +// CONSOLE +// TEARDOWN + +[source,js] +-------------------------------------------------- +GET /follower_index/_ccr/info?filter_path=follower_indices.parameters +-------------------------------------------------- +// CONSOLE + +////////////////////////// + +The following output from the follow info api describes all the default +values for the above described index follow request parameters: + +[source,js] +-------------------------------------------------- +{ + "follower_indices" : [ + { + "parameters" : { + "max_read_request_operation_count" : 5120, + "max_read_request_size" : "32mb", + "max_outstanding_read_requests" : 12, + "max_write_request_operation_count" : 5120, + "max_write_request_size" : "9223372036854775807b", + "max_outstanding_write_requests" : 9, + "max_write_buffer_count" : 2147483647, + "max_write_buffer_size" : "512mb", + "max_retry_delay" : "500ms", + "read_poll_timeout" : "1m" + } + } + ] +} + +-------------------------------------------------- +// TESTRESPONSE \ No newline at end of file From 4dee3f741846ee8e7ccc59c09c550e0c88f10039 Mon Sep 17 00:00:00 2001 From: Tim Vernum Date: Wed, 30 Jan 2019 20:13:48 +1100 Subject: [PATCH 039/100] Add classifier to tar.gz in docker compose (#38011) The distribution now includes a platform specific classifier that the docker build wasn't taking into account. Relates: #37881 --- distribution/docker/build.gradle | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 8a5cea936bbcd..e4d27da1f1fb0 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -18,8 +18,9 @@ dependencies { } ext.expansions = { oss -> + String classifier = 'linux-x86_64' return [ - 'elasticsearch' : oss ? "elasticsearch-oss-${VersionProperties.elasticsearch}.tar.gz" : "elasticsearch-${VersionProperties.elasticsearch}.tar.gz", + 'elasticsearch' : oss ? "elasticsearch-oss-${VersionProperties.elasticsearch}-${classifier}.tar.gz" : "elasticsearch-${VersionProperties.elasticsearch}-${classifier}.tar.gz", 'jdkUrl' : 'https://download.java.net/java/GA/jdk11/9/GPL/openjdk-11.0.2_linux-x64_bin.tar.gz', 'jdkVersion' : '11.0.2', 'license': oss ? 'Apache-2.0' : 'Elastic License', From b63b50b945d4c44c70d56a9ce2ba9a25dd897147 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 30 Jan 2019 10:28:24 +0100 Subject: [PATCH 040/100] Give precedence to index creation when mixing typed templates with typeless index creation and vice-versa. (#37871) Currently if you mix typed templates and typeless index creation or typeless templates and typed index creation then you will end up with an error because Elasticsearch tries to create an index that has multiple types: `_doc` and the explicit type name that you used. This commit proposes to give precedence to the index creation call so that the type from the template will be ignored if the index creation call is typeless while the template is typed, and the type from the index creation call will be used if there is a typeless template. This is consistent with the fact that index creation already "wins" if a field is defined differently in the index creation call and in a template: the definition from the index creation call is used in such cases. Closes #37773 --- .../mapping/removal_of_types.asciidoc | 74 ++++++++++ .../20_mix_typeless_typeful.yml | 137 ++++++++++++++++++ .../metadata/MetaDataCreateIndexService.java | 22 +++ .../metadata/IndexCreationTaskTests.java | 26 ++++ 4 files changed, 259 insertions(+) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/20_mix_typeless_typeful.yml diff --git a/docs/reference/mapping/removal_of_types.asciidoc b/docs/reference/mapping/removal_of_types.asciidoc index ee5ee4b4fe664..b9066a4c7af49 100644 --- a/docs/reference/mapping/removal_of_types.asciidoc +++ b/docs/reference/mapping/removal_of_types.asciidoc @@ -534,3 +534,77 @@ PUT index/_doc/1 The <>, <>, <> and <> APIs will continue to return a `_type` key in the response in 7.0, but it is considered deprecated and will be removed in 8.0. + +[float] +=== Index templates + +It is recommended to make index templates typeless before upgrading to 7.0 by +re-adding them with `include_type_name` set to `false`. + +In case typeless templates are used with typed index creation calls or typed +templates are used with typeless index creation calls, the template will still +be applied but the index creation call decides whether there should be a type +or not. For instance in the below example, `index-1-01` will have a type in +spite of the fact that it matches a template that is typeless, and `index-2-01` +will be typeless in spite of the fact that it matches a template that defines +a type. Both `index-1-01` and `index-2-01` will inherit the `foo` field from +the template that they match. + +[source,js] +-------------------------------------------------- +PUT _template/template1 +{ + "index_patterns":[ "index-1-*" ], + "mappings": { + "properties": { + "foo": { + "type": "keyword" + } + } + } +} + +PUT _template/template2?include_type_name=true +{ + "index_patterns":[ "index-2-*" ], + "mappings": { + "type": { + "properties": { + "foo": { + "type": "keyword" + } + } + } + } +} + +PUT index-1-01?include_type_name=true +{ + "mappings": { + "type": { + "properties": { + "bar": { + "type": "long" + } + } + } + } +} + +PUT index-2-01 +{ + "mappings": { + "properties": { + "bar": { + "type": "long" + } + } + } +} +-------------------------------------------------- +// CONSOLE + +In case of implicit index creation, because of documents that get indexed in +an index that doesn't exist yet, the template is always honored. This is +usually not a problem due to the fact that typless index calls work on typed +indices. diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/20_mix_typeless_typeful.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/20_mix_typeless_typeful.yml new file mode 100644 index 0000000000000..d196d5e5dd8c2 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/20_mix_typeless_typeful.yml @@ -0,0 +1,137 @@ +--- +"Create a typeless index while there is a typed template": + + - skip: + version: " - 6.99.99" + reason: needs change to be backported to 6.7 + + - do: + indices.put_template: + include_type_name: true + name: test_template + body: + index_patterns: test-* + mappings: + my_type: + properties: + foo: + type: keyword + + - do: + indices.create: + include_type_name: false + index: test-1 + body: + mappings: + properties: + bar: + type: "long" + + - do: + indices.get_mapping: + include_type_name: true + index: test-1 + + - is_true: test-1.mappings._doc # the index creation call won + - is_false: test-1.mappings.my_type + - is_true: test-1.mappings._doc.properties.foo + - is_true: test-1.mappings._doc.properties.bar + +--- +"Create a typed index while there is a typeless template": + + - skip: + version: " - 6.99.99" + reason: needs change to be backported to 6.7 + + - do: + indices.put_template: + include_type_name: false + name: test_template + body: + index_patterns: test-* + mappings: + properties: + foo: + type: keyword + + - do: + indices.create: + include_type_name: true + index: test-1 + body: + mappings: + my_type: + properties: + bar: + type: "long" + + - do: + indices.get_mapping: + include_type_name: true + index: test-1 + + - is_true: test-1.mappings.my_type # the index creation call won + - is_false: test-1.mappings._doc + - is_true: test-1.mappings.my_type.properties.foo + - is_true: test-1.mappings.my_type.properties.bar + +--- +"Implicitly create a typed index while there is a typeless template": + + - skip: + version: " - 6.99.99" + reason: needs change to be backported to 6.7 + + - do: + indices.put_template: + include_type_name: false + name: test_template + body: + index_patterns: test-* + mappings: + properties: + foo: + type: keyword + + - do: + catch: /the final mapping would have more than 1 type/ + index: + index: test-1 + type: my_type + body: { bar: 42 } + +--- +"Implicitly create a typeless index while there is a typed template": + + - skip: + version: " - 6.99.99" + reason: needs typeless index operations to work on typed indices + + - do: + indices.put_template: + include_type_name: true + name: test_template + body: + index_patterns: test-* + mappings: + my_type: + properties: + foo: + type: keyword + + - do: + index: + index: test-1 + type: my_type + body: { bar: 42 } + + - do: + indices.get_mapping: + include_type_name: true + index: test-1 + + - is_true: test-1.mappings.my_type # the template is honored + - is_false: test-1.mappings._doc + - is_true: test-1.mappings.my_type.properties.foo + - is_true: test-1.mappings.my_type.properties.bar diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index d9120342cf4cd..838b1e2547204 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -318,6 +318,28 @@ public ClusterState execute(ClusterState currentState) throws Exception { if (mappings.containsKey(cursor.key)) { XContentHelper.mergeDefaults(mappings.get(cursor.key), MapperService.parseMapping(xContentRegistry, mappingString)); + } else if (mappings.size() == 1 && cursor.key.equals(MapperService.SINGLE_MAPPING_NAME)) { + // Typeless template with typed mapping + Map templateMapping = MapperService.parseMapping(xContentRegistry, mappingString); + assert templateMapping.size() == 1 : templateMapping; + assert cursor.key.equals(templateMapping.keySet().iterator().next()) : + cursor.key + " != " + templateMapping; + Map.Entry> mappingEntry = mappings.entrySet().iterator().next(); + templateMapping = Collections.singletonMap( + mappingEntry.getKey(), // reuse type name from the mapping + templateMapping.values().iterator().next()); // but actual mappings from the template + XContentHelper.mergeDefaults(mappingEntry.getValue(), templateMapping); + } else if (template.mappings().size() == 1 && mappings.containsKey(MapperService.SINGLE_MAPPING_NAME)) { + // Typed template with typeless mapping + Map templateMapping = MapperService.parseMapping(xContentRegistry, mappingString); + assert templateMapping.size() == 1 : templateMapping; + assert cursor.key.equals(templateMapping.keySet().iterator().next()) : + cursor.key + " != " + templateMapping; + Map mapping = mappings.get(MapperService.SINGLE_MAPPING_NAME); + templateMapping = Collections.singletonMap( + MapperService.SINGLE_MAPPING_NAME, // make template mapping typeless + templateMapping.values().iterator().next()); + XContentHelper.mergeDefaults(mapping, templateMapping); } else { mappings.put(cursor.key, MapperService.parseMapping(xContentRegistry, mappingString)); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java index 518a60ffe38f6..f2a87d09eb1f2 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java @@ -310,6 +310,32 @@ public void testWriteIndexValidationException() throws Exception { assertThat(exception.getMessage(), startsWith("alias [alias1] has more than one write index [")); } + public void testTypelessTemplateWithTypedIndexCreation() throws Exception { + addMatchingTemplate(builder -> builder.putMapping("type", "{\"type\": {}}")); + setupRequestMapping(MapperService.SINGLE_MAPPING_NAME, new CompressedXContent("{\"_doc\":{}}")); + executeTask(); + assertThat(getMappingsFromResponse(), Matchers.hasKey(MapperService.SINGLE_MAPPING_NAME)); + } + + public void testTypedTemplateWithTypelessIndexCreation() throws Exception { + addMatchingTemplate(builder -> builder.putMapping(MapperService.SINGLE_MAPPING_NAME, "{\"_doc\": {}}")); + setupRequestMapping("type", new CompressedXContent("{\"type\":{}}")); + executeTask(); + assertThat(getMappingsFromResponse(), Matchers.hasKey("type")); + } + + public void testTypedTemplate() throws Exception { + addMatchingTemplate(builder -> builder.putMapping("type", "{\"type\": {}}")); + executeTask(); + assertThat(getMappingsFromResponse(), Matchers.hasKey("type")); + } + + public void testTypelessTemplate() throws Exception { + addMatchingTemplate(builder -> builder.putMapping(MapperService.SINGLE_MAPPING_NAME, "{\"_doc\": {}}")); + executeTask(); + assertThat(getMappingsFromResponse(), Matchers.hasKey(MapperService.SINGLE_MAPPING_NAME)); + } + private IndexRoutingTable createIndexRoutingTableWithStartedShards(Index index) { final IndexRoutingTable idxRoutingTable = mock(IndexRoutingTable.class); From c8af0f4bfafb1ec15a59b91b0e04f157a503d615 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 30 Jan 2019 10:31:51 +0100 Subject: [PATCH 041/100] Use mappings to format doc-value fields by default. (#30831) Doc-value fields now return a value that is based on the mappings rather than the script implementation by default. This deprecates the special `use_field_mapping` docvalue format which was added in #29639 only to ease the transition to 7.x and it is not necessary anymore in 7.0. --- .../migration/migrate_7_0/search.asciidoc | 10 ++++ .../search/request/docvalue-fields.asciidoc | 23 +++++---- .../search/request/inner-hits.asciidoc | 5 +- .../sql/endpoints/translate.asciidoc | 3 +- .../upgrades/QueryBuilderBWCIT.java | 2 +- .../test/search.inner_hits/10_basic.yml | 8 ++-- .../test/search/10_source_filtering.yml | 28 +++++------ .../rest-api-spec/test/search/30_limits.yml | 13 ++--- .../fetch/subphase/DocValueFieldsContext.java | 2 - .../subphase/DocValueFieldsFetchSubPhase.java | 47 +++++++------------ .../index/query/InnerHitBuilderTests.java | 7 +-- .../search/fields/SearchFieldsIT.java | 24 ++++------ .../extractor/fields/ExtractedField.java | 3 +- .../persistence/DatafeedConfigProvider.java | 5 +- .../ml/job/persistence/JobConfigProvider.java | 13 +++-- .../extractor/fields/ExtractedFieldTests.java | 3 +- .../fields/ExtractedFieldsTests.java | 3 +- .../fields/TimeBasedExtractedFieldsTests.java | 3 +- .../sql/qa/single_node/CliExplainIT.java | 1 - .../sql/action/SqlTranslateResponseTests.java | 3 +- .../execution/search/SqlSourceBuilder.java | 4 +- .../querydsl/container/QueryContainer.java | 3 +- .../xpack/sql/action/SqlLicenseIT.java | 2 +- .../sql/action/SqlTranslateActionIT.java | 2 +- .../container/QueryContainerTests.java | 3 +- .../sql/querydsl/query/BoolQueryTests.java | 10 ++-- .../sql/querydsl/query/LeafQueryTests.java | 4 +- .../sql/querydsl/query/NestedQueryTests.java | 17 ++++--- .../rest-api-spec/test/sql/translate.yml | 6 +-- 29 files changed, 105 insertions(+), 152 deletions(-) diff --git a/docs/reference/migration/migrate_7_0/search.asciidoc b/docs/reference/migration/migrate_7_0/search.asciidoc index 67adf9363406c..0f3dcf9771c3d 100644 --- a/docs/reference/migration/migrate_7_0/search.asciidoc +++ b/docs/reference/migration/migrate_7_0/search.asciidoc @@ -122,6 +122,16 @@ using the "all fields" mode ("default_field": "*") or other fieldname expansions Search requests with extra content after the main object will no longer be accepted by the `_search` endpoint. A parsing exception will be thrown instead. +[float] +==== Doc-value fields default format + +The format of doc-value fields is changing to be the same as what could be +obtained in 6.x with the special `use_field_mapping` format. This is mostly a +change for date fields, which are now formatted based on the format that is +configured in the mappings by default. This behavior can be changed by +specifying a <> within the doc-value +field. + [float] ==== Context Completion Suggester diff --git a/docs/reference/search/request/docvalue-fields.asciidoc b/docs/reference/search/request/docvalue-fields.asciidoc index bcfcb20d1d53b..6697b5bb3e383 100644 --- a/docs/reference/search/request/docvalue-fields.asciidoc +++ b/docs/reference/search/request/docvalue-fields.asciidoc @@ -12,9 +12,9 @@ GET /_search "match_all": {} }, "docvalue_fields" : [ + "my_ip_field", <1> { - "field": "my_ip_field", <1> - "format": "use_field_mapping" <2> + "field": "my_keyword_field" <2> }, { "field": "my_date_field", @@ -25,10 +25,10 @@ GET /_search -------------------------------------------------- // CONSOLE <1> the name of the field -<2> the special `use_field_mapping` format tells Elasticsearch to use the format from the mapping -<3> date fields may use a custom format +<2> an object notation is supported as well +<3> the object notation allows to specify a custom format -Doc value fields can work on fields that are not stored. +Doc value fields can work on fields that have doc-values enabled, regardless of whether they are stored `*` can be used as a wild card, for example: @@ -41,8 +41,8 @@ GET /_search }, "docvalue_fields" : [ { - "field": "*field", <1> - "format": "use_field_mapping" <2> + "field": "*_date_field", <1> + "format": "epoch_millis" <2> } ] } @@ -62,9 +62,8 @@ While most fields do not support custom formats, some of them do: - <> fields can take any <>. - <> fields accept a https://docs.oracle.com/javase/8/docs/api/java/text/DecimalFormat.html[DecimalFormat pattern]. -All fields support the special `use_field_mapping` format, which tells -Elasticsearch to use the mappings to figure out a default format. +By default fields are formatted based on a sensible configuration that depends +on their mappings: `long`, `double` and other numeric fields are formatted as +numbers, `keyword` fields are formatted as strings, `date` fields are formatted +with the configured `date` format, etc. -NOTE: The default is currently to return the same output as -<>. However it will change in 7.0 -to behave as if the `use_field_mapping` format was provided. diff --git a/docs/reference/search/request/inner-hits.asciidoc b/docs/reference/search/request/inner-hits.asciidoc index b287b1609703e..7774e34c2c00d 100644 --- a/docs/reference/search/request/inner-hits.asciidoc +++ b/docs/reference/search/request/inner-hits.asciidoc @@ -246,10 +246,7 @@ POST test/_search "inner_hits": { "_source" : false, "docvalue_fields" : [ - { - "field": "comments.text.keyword", - "format": "use_field_mapping" - } + "comments.text.keyword" ] } } diff --git a/docs/reference/sql/endpoints/translate.asciidoc b/docs/reference/sql/endpoints/translate.asciidoc index 2bb7bfae4d225..de9d8adbeab0b 100644 --- a/docs/reference/sql/endpoints/translate.asciidoc +++ b/docs/reference/sql/endpoints/translate.asciidoc @@ -27,8 +27,7 @@ Which returns: "size" : 10, "docvalue_fields" : [ { - "field": "page_count", - "format": "use_field_mapping" + "field": "page_count" }, { "field": "release_date", diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java index 10bdcc234c656..f22b1b44c0763 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java @@ -201,7 +201,7 @@ public void testQueryBuilderBWC() throws Exception { QueryBuilder expectedQueryBuilder = (QueryBuilder) CANDIDATES.get(i)[1]; Request request = new Request("GET", "/" + index + "/_search"); request.setJsonEntity("{\"query\": {\"ids\": {\"values\": [\"" + Integer.toString(i) + "\"]}}, " + - "\"docvalue_fields\": [{\"field\":\"query.query_builder_field\", \"format\":\"use_field_mapping\"}]}"); + "\"docvalue_fields\": [{\"field\":\"query.query_builder_field\"}]}"); Response rsp = client().performRequest(request); assertEquals(200, rsp.getStatusLine().getStatusCode()); Map hitRsp = (Map) ((List) ((Map)toMap(rsp).get("hits")).get("hits")).get(0); diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml index c5bfca5c5b1d7..e0d61192bf754 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml @@ -46,8 +46,8 @@ setup: "Nested doc version and seqIDs": - skip: - version: " - 6.3.99" - reason: "object notation for docvalue_fields was introduced in 6.4" + version: " - 6.99.99" + reason: "Triggers warnings before 7.0" - do: index: @@ -62,7 +62,7 @@ setup: - do: search: rest_total_hits_as_int: true - body: { "query" : { "nested" : { "path" : "nested_field", "query" : { "match_all" : {} }, "inner_hits" : { version: true, "docvalue_fields": [ { "field": "_seq_no", "format": "use_field_mapping" } ]} }}, "version": true, "docvalue_fields" : [ { "field": "_seq_no", "format": "use_field_mapping" } ] } + body: { "query" : { "nested" : { "path" : "nested_field", "query" : { "match_all" : {} }, "inner_hits" : { version: true, "docvalue_fields": [ "_seq_no" ]} }}, "version": true, "docvalue_fields" : [ "_seq_no" ] } - match: { hits.total: 1 } - match: { hits.hits.0._index: "test" } @@ -86,7 +86,7 @@ setup: - do: search: rest_total_hits_as_int: true - body: { "query" : { "nested" : { "path" : "nested_field", "query" : { "match_all" : {} }, "inner_hits" : { version: true, "docvalue_fields": [ { "field": "_seq_no", "format": "use_field_mapping" } ]} }}, "version": true, "docvalue_fields" : [ { "field": "_seq_no", "format": "use_field_mapping" } ] } + body: { "query" : { "nested" : { "path" : "nested_field", "query" : { "match_all" : {} }, "inner_hits" : { version: true, "docvalue_fields": [ "_seq_no" ]} }}, "version": true, "docvalue_fields" : [ "_seq_no" ] } - match: { hits.total: 1 } - match: { hits.hits.0._index: "test" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yml index e5277c0edcbbb..18191c5ee3ac2 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yml @@ -144,12 +144,9 @@ setup: --- "docvalue_fields": - skip: - version: " - 6.4.0" - reason: format option was added in 6.4 and the deprecation message changed in 6.4.1 - features: warnings + version: " - 6.9.99" + reason: Triggers a deprecation warning before 7.0 - do: - warnings: - - 'There are doc-value fields which are not using a format. The output will change in 7.0 when doc value fields get formatted based on mappings by default. It is recommended to pass [format=use_field_mapping] with a doc value field in order to opt in for the future behaviour and ease the migration to 7.0: [count]' search: body: docvalue_fields: [ "count" ] @@ -158,12 +155,9 @@ setup: --- "multiple docvalue_fields": - skip: - version: " - 6.4.0" - reason: format option was added in 6.4 and the deprecation message changed in 6.4.1 - features: warnings + version: " - 6.9.99" + reason: Triggered a deprecation warning before 7.0 - do: - warnings: - - 'There are doc-value fields which are not using a format. The output will change in 7.0 when doc value fields get formatted based on mappings by default. It is recommended to pass [format=use_field_mapping] with a doc value field in order to opt in for the future behaviour and ease the migration to 7.0: [count, include.field1.keyword]' search: body: docvalue_fields: [ "count", "include.field1.keyword" ] @@ -172,12 +166,9 @@ setup: --- "docvalue_fields as url param": - skip: - version: " - 6.4.0" - reason: format option was added in 6.4 and the deprecation message changed in 6.4.1 - features: warnings + version: " - 6.99.99" + reason: Triggered a deprecation warning before 7.0 - do: - warnings: - - 'There are doc-value fields which are not using a format. The output will change in 7.0 when doc value fields get formatted based on mappings by default. It is recommended to pass [format=use_field_mapping] with a doc value field in order to opt in for the future behaviour and ease the migration to 7.0: [count]' search: docvalue_fields: [ "count" ] - match: { hits.hits.0.fields.count: [1] } @@ -185,9 +176,12 @@ setup: --- "docvalue_fields with default format": - skip: - version: " - 6.3.99" - reason: format option was added in 6.4 + version: " - 6.99.99" + reason: Only triggers warnings on 7.0+ + features: warnings - do: + warnings: + - "[use_field_mapping] is a special format that was only used to ease the transition to 7.x. It has become the default and shouldn't be set explicitly anymore." search: body: docvalue_fields: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml index 18a004077aeae..07a871d37e65f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml @@ -67,8 +67,8 @@ setup: "Docvalues_fields size limit": - skip: - version: " - 6.3.99" - reason: "The object notation for docvalue_fields is only supported on 6.4+" + version: " - 6.99.99" + reason: "Triggers warnings before 7.0" - do: catch: /Trying to retrieve too many docvalue_fields\. Must be less than or equal to[:] \[2\] but was \[3\]\. This limit can be set by changing the \[index.max_docvalue_fields_search\] index level setting\./ search: @@ -78,12 +78,9 @@ setup: query: match_all: {} docvalue_fields: - - field: "one" - format: "use_field_mapping" - - field: "two" - format: "use_field_mapping" - - field: "three" - format: "use_field_mapping" + - "one" + - "two" + - "three" --- "Script_fields size limit": diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsContext.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsContext.java index cf1596fd326b9..daafe0970290b 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsContext.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsContext.java @@ -38,8 +38,6 @@ */ public class DocValueFieldsContext { - public static final String USE_DEFAULT_FORMAT = "use_field_mapping"; - /** * Wrapper around a field name and the format that should be used to * display values of this field. diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java index eae3188d865b8..e94ac0fdf6c99 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java @@ -28,7 +28,6 @@ import org.elasticsearch.index.fielddata.AtomicNumericFieldData; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; -import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.mapper.MappedFieldType; @@ -46,7 +45,6 @@ import java.util.HashMap; import java.util.List; import java.util.Objects; -import java.util.stream.Collectors; /** * Query sub phase which pulls data from doc values @@ -55,7 +53,8 @@ */ public final class DocValueFieldsFetchSubPhase implements FetchSubPhase { - private static final DeprecationLogger deprecationLogger = new DeprecationLogger( + private static final String USE_DEFAULT_FORMAT = "use_field_mapping"; + private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger( LogManager.getLogger(DocValueFieldsFetchSubPhase.class)); @Override @@ -66,9 +65,9 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept String name = context.collapse().getFieldName(); if (context.docValueFieldsContext() == null) { context.docValueFieldsContext(new DocValueFieldsContext( - Collections.singletonList(new FieldAndFormat(name, DocValueFieldsContext.USE_DEFAULT_FORMAT)))); + Collections.singletonList(new FieldAndFormat(name, null)))); } else if (context.docValueFieldsContext().fields().stream().map(ff -> ff.field).anyMatch(name::equals) == false) { - context.docValueFieldsContext().fields().add(new FieldAndFormat(name, DocValueFieldsContext.USE_DEFAULT_FORMAT)); + context.docValueFieldsContext().fields().add(new FieldAndFormat(name, null)); } } @@ -79,13 +78,13 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept hits = hits.clone(); // don't modify the incoming hits Arrays.sort(hits, Comparator.comparingInt(SearchHit::docId)); - List noFormatFields = context.docValueFieldsContext().fields().stream().filter(f -> f.format == null).map(f -> f.field) - .collect(Collectors.toList()); - if (noFormatFields.isEmpty() == false) { - deprecationLogger.deprecated("There are doc-value fields which are not using a format. The output will " - + "change in 7.0 when doc value fields get formatted based on mappings by default. It is recommended to pass " - + "[format={}] with a doc value field in order to opt in for the future behaviour and ease the migration to " - + "7.0: {}", DocValueFieldsContext.USE_DEFAULT_FORMAT, noFormatFields); + if (context.docValueFieldsContext().fields().stream() + .map(f -> f.format) + .filter(USE_DEFAULT_FORMAT::equals) + .findAny() + .isPresent()) { + DEPRECATION_LOGGER.deprecated("[" + USE_DEFAULT_FORMAT + "] is a special format that was only used to " + + "ease the transition to 7.x. It has become the default and shouldn't be set explicitly anymore."); } for (FieldAndFormat fieldAndFormat : context.docValueFieldsContext().fields()) { @@ -93,19 +92,14 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept MappedFieldType fieldType = context.mapperService().fullName(field); if (fieldType != null) { final IndexFieldData indexFieldData = context.getForField(fieldType); - final DocValueFormat format; - if (fieldAndFormat.format == null) { - format = null; - } else { - String formatDesc = fieldAndFormat.format; - if (Objects.equals(formatDesc, DocValueFieldsContext.USE_DEFAULT_FORMAT)) { - formatDesc = null; - } - format = fieldType.docValueFormat(formatDesc, null); + String formatDesc = fieldAndFormat.format; + if (Objects.equals(formatDesc, USE_DEFAULT_FORMAT)) { + // TODO: Remove in 8.x + formatDesc = null; } + final DocValueFormat format = fieldType.docValueFormat(formatDesc, null); LeafReaderContext subReaderContext = null; AtomicFieldData data = null; - ScriptDocValues scriptValues = null; // legacy SortedBinaryDocValues binaryValues = null; // binary / string / ip fields SortedNumericDocValues longValues = null; // int / date fields SortedNumericDoubleValues doubleValues = null; // floating-point fields @@ -115,9 +109,7 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept int readerIndex = ReaderUtil.subIndex(hit.docId(), context.searcher().getIndexReader().leaves()); subReaderContext = context.searcher().getIndexReader().leaves().get(readerIndex); data = indexFieldData.load(subReaderContext); - if (format == null) { - scriptValues = data.getLegacyFieldValues(); - } else if (indexFieldData instanceof IndexNumericFieldData) { + if (indexFieldData instanceof IndexNumericFieldData) { if (((IndexNumericFieldData) indexFieldData).getNumericType().isFloatingPoint()) { doubleValues = ((AtomicNumericFieldData) data).getDoubleValues(); } else { @@ -138,10 +130,7 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept final List values = hitField.getValues(); int subDocId = hit.docId() - subReaderContext.docBase; - if (scriptValues != null) { - scriptValues.setNextDocId(subDocId); - values.addAll(scriptValues); - } else if (binaryValues != null) { + if (binaryValues != null) { if (binaryValues.advanceExact(subDocId)) { for (int i = 0, count = binaryValues.docValueCount(); i < count; ++i) { values.add(format.format(binaryValues.nextValue())); diff --git a/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java index 257ee807419b6..54d478a7f6aec 100644 --- a/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java @@ -32,7 +32,6 @@ import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext.FieldAndFormat; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilderTests; @@ -158,8 +157,7 @@ public static InnerHitBuilder randomInnerHits() { innerHits.setStoredFieldNames(randomListStuff(16, () -> randomAlphaOfLengthBetween(1, 16))); } innerHits.setDocValueFields(randomListStuff(16, - () -> new FieldAndFormat(randomAlphaOfLengthBetween(1, 16), - randomBoolean() ? null : DocValueFieldsContext.USE_DEFAULT_FORMAT))); + () -> new FieldAndFormat(randomAlphaOfLengthBetween(1, 16), null))); // Random script fields deduped on their field name. Map scriptFields = new HashMap<>(); for (SearchSourceBuilder.ScriptField field: randomListStuff(16, InnerHitBuilderTests::randomScript)) { @@ -201,8 +199,7 @@ static InnerHitBuilder mutate(InnerHitBuilder original) throws IOException { modifiers.add(() -> { if (randomBoolean()) { copy.setDocValueFields(randomValueOtherThan(copy.getDocValueFields(), - () -> randomListStuff(16, () -> new FieldAndFormat(randomAlphaOfLengthBetween(1, 16), - randomBoolean() ? null : DocValueFieldsContext.USE_DEFAULT_FORMAT)))); + () -> randomListStuff(16, () -> new FieldAndFormat(randomAlphaOfLengthBetween(1, 16), null)))); } else { copy.addDocValueField(randomAlphaOfLengthBetween(1, 16)); } diff --git a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java index fc69df5987aff..81315071273e0 100644 --- a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java +++ b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java @@ -19,7 +19,6 @@ package org.elasticsearch.search.fields; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; @@ -49,7 +48,6 @@ import org.elasticsearch.test.InternalSettingsPlugin; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; -import org.joda.time.ReadableDateTime; import org.joda.time.format.DateTimeFormat; import java.time.ZoneOffset; @@ -804,13 +802,12 @@ public void testDocValueFields() throws Exception { assertThat(searchResponse.getHits().getAt(0).getFields().get("long_field").getValue(), equalTo((Object) 4L)); assertThat(searchResponse.getHits().getAt(0).getFields().get("float_field").getValue(), equalTo((Object) 5.0)); assertThat(searchResponse.getHits().getAt(0).getFields().get("double_field").getValue(), equalTo((Object) 6.0d)); - DateTime dateField = searchResponse.getHits().getAt(0).getFields().get("date_field").getValue(); - assertThat(dateField.getMillis(), equalTo(date.toInstant().toEpochMilli())); + assertThat(searchResponse.getHits().getAt(0).getFields().get("date_field").getValue(), + equalTo(DateFormatter.forPattern("dateOptionalTime").format(date))); assertThat(searchResponse.getHits().getAt(0).getFields().get("boolean_field").getValue(), equalTo((Object) true)); assertThat(searchResponse.getHits().getAt(0).getFields().get("text_field").getValue(), equalTo("foo")); assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo")); - assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(), - equalTo(new BytesRef(new byte[] {42, 100}))); + assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(), equalTo("KmQ")); assertThat(searchResponse.getHits().getAt(0).getFields().get("ip_field").getValue(), equalTo("::1")); builder = client().prepareSearch().setQuery(matchAllQuery()) @@ -830,13 +827,12 @@ public void testDocValueFields() throws Exception { assertThat(searchResponse.getHits().getAt(0).getFields().get("long_field").getValue(), equalTo((Object) 4L)); assertThat(searchResponse.getHits().getAt(0).getFields().get("float_field").getValue(), equalTo((Object) 5.0)); assertThat(searchResponse.getHits().getAt(0).getFields().get("double_field").getValue(), equalTo((Object) 6.0d)); - dateField = searchResponse.getHits().getAt(0).getFields().get("date_field").getValue(); - assertThat(dateField.getMillis(), equalTo(date.toInstant().toEpochMilli())); + assertThat(searchResponse.getHits().getAt(0).getFields().get("date_field").getValue(), + equalTo(DateFormatter.forPattern("dateOptionalTime").format(date))); assertThat(searchResponse.getHits().getAt(0).getFields().get("boolean_field").getValue(), equalTo((Object) true)); assertThat(searchResponse.getHits().getAt(0).getFields().get("text_field").getValue(), equalTo("foo")); assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo")); - assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(), - equalTo(new BytesRef(new byte[] {42, 100}))); + assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(), equalTo("KmQ")); assertThat(searchResponse.getHits().getAt(0).getFields().get("ip_field").getValue(), equalTo("::1")); builder = client().prepareSearch().setQuery(matchAllQuery()) @@ -1001,9 +997,7 @@ public void testDocValueFieldsWithFieldAlias() throws Exception { DocumentField dateField = fields.get("date_field"); assertThat(dateField.getName(), equalTo("date_field")); - - ReadableDateTime fetchedDate = dateField.getValue(); - assertThat(fetchedDate.getMillis(), equalTo(date.toInstant().getMillis())); + assertThat(dateField.getValue(), equalTo("1990-12-29")); } public void testWildcardDocValueFieldsWithFieldAlias() throws Exception { @@ -1065,9 +1059,7 @@ public void testWildcardDocValueFieldsWithFieldAlias() throws Exception { DocumentField dateField = fields.get("date_field"); assertThat(dateField.getName(), equalTo("date_field")); - - ReadableDateTime fetchedDate = dateField.getValue(); - assertThat(fetchedDate.getMillis(), equalTo(date.toInstant().getMillis())); + assertThat(dateField.getValue(), equalTo("1990-12-29")); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedField.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedField.java index 4223bff49825e..451f480b27830 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedField.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedField.java @@ -7,7 +7,6 @@ import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import java.util.List; import java.util.Map; @@ -52,7 +51,7 @@ public ExtractionMethod getExtractionMethod() { public abstract Object[] value(SearchHit hit); public String getDocValueFormat() { - return DocValueFieldsContext.USE_DEFAULT_FORMAT; + return null; } public static ExtractedField newTimeField(String name, ExtractionMethod extractionMethod) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java index d9ea6cb7c32e4..36e71de8bcba1 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java @@ -44,7 +44,6 @@ import org.elasticsearch.index.query.WildcardQueryBuilder; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedUpdate; @@ -198,7 +197,7 @@ public void onFailure(Exception e) { public void findDatafeedsForJobIds(Collection jobIds, ActionListener> listener) { SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildDatafeedJobIdsQuery(jobIds)); sourceBuilder.fetchSource(false); - sourceBuilder.docValueField(DatafeedConfig.ID.getPreferredName(), DocValueFieldsContext.USE_DEFAULT_FORMAT); + sourceBuilder.docValueField(DatafeedConfig.ID.getPreferredName(), null); SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) .setIndicesOptions(IndicesOptions.lenientExpandOpen()) @@ -366,7 +365,7 @@ public void expandDatafeedIds(String expression, boolean allowNoDatafeeds, Actio SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildDatafeedIdQuery(tokens)); sourceBuilder.sort(DatafeedConfig.ID.getPreferredName()); sourceBuilder.fetchSource(false); - sourceBuilder.docValueField(DatafeedConfig.ID.getPreferredName(), DocValueFieldsContext.USE_DEFAULT_FORMAT); + sourceBuilder.docValueField(DatafeedConfig.ID.getPreferredName(), null); SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) .setIndicesOptions(IndicesOptions.lenientExpandOpen()) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java index a21e6bc8c133c..9019dc2032ccf 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java @@ -52,7 +52,6 @@ import org.elasticsearch.index.query.WildcardQueryBuilder; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; @@ -424,7 +423,7 @@ public void jobIdMatches(List ids, ActionListener> listener SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(boolQueryBuilder); sourceBuilder.fetchSource(false); - sourceBuilder.docValueField(Job.ID.getPreferredName(), DocValueFieldsContext.USE_DEFAULT_FORMAT); + sourceBuilder.docValueField(Job.ID.getPreferredName(), null); SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) .setIndicesOptions(IndicesOptions.lenientExpandOpen()) @@ -509,8 +508,8 @@ public void expandJobsIds(String expression, boolean allowNoJobs, boolean exclud SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildQuery(tokens, excludeDeleting)); sourceBuilder.sort(Job.ID.getPreferredName()); sourceBuilder.fetchSource(false); - sourceBuilder.docValueField(Job.ID.getPreferredName(), DocValueFieldsContext.USE_DEFAULT_FORMAT); - sourceBuilder.docValueField(Job.GROUPS.getPreferredName(), DocValueFieldsContext.USE_DEFAULT_FORMAT); + sourceBuilder.docValueField(Job.ID.getPreferredName(), null); + sourceBuilder.docValueField(Job.GROUPS.getPreferredName(), null); SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) .setIndicesOptions(IndicesOptions.lenientExpandOpen()) @@ -554,8 +553,8 @@ private SearchRequest makeExpandIdsSearchRequest(String expression, boolean excl SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildQuery(tokens, excludeDeleting)); sourceBuilder.sort(Job.ID.getPreferredName()); sourceBuilder.fetchSource(false); - sourceBuilder.docValueField(Job.ID.getPreferredName(), DocValueFieldsContext.USE_DEFAULT_FORMAT); - sourceBuilder.docValueField(Job.GROUPS.getPreferredName(), DocValueFieldsContext.USE_DEFAULT_FORMAT); + sourceBuilder.docValueField(Job.ID.getPreferredName(), null); + sourceBuilder.docValueField(Job.GROUPS.getPreferredName(), null); return client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) .setIndicesOptions(IndicesOptions.lenientExpandOpen()) @@ -638,7 +637,7 @@ public void expandGroupIds(List groupIds, ActionListener sourceBuilder.docValueField(field.field, - field.format == null ? DocValueFieldsContext.USE_DEFAULT_FORMAT : field.format)); + docFields.forEach(field -> sourceBuilder.docValueField(field.field, field.format)); scriptFields.forEach(sourceBuilder::scriptField); } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java index 9a784b7b112ff..fee8d0e942a3b 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java @@ -11,7 +11,6 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; import org.elasticsearch.xpack.sql.execution.search.FieldExtraction; import org.elasticsearch.xpack.sql.execution.search.SourceGenerator; @@ -183,7 +182,7 @@ private Tuple nestedHitFieldRef(FieldAttribute List nestedRefs = new ArrayList<>(); String name = aliasName(attr); - String format = attr.field().getDataType() == DataType.DATETIME ? "epoch_millis" : DocValueFieldsContext.USE_DEFAULT_FORMAT; + String format = attr.field().getDataType() == DataType.DATETIME ? "epoch_millis" : null; Query q = rewriteToContainNestedField(query, attr.source(), attr.nestedParent().name(), name, format, attr.field().isAggregatable()); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlLicenseIT.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlLicenseIT.java index 50dda656ab4d1..9e56f46949ff6 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlLicenseIT.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlLicenseIT.java @@ -154,7 +154,7 @@ public void testSqlTranslateActionLicense() throws Exception { .query("SELECT * FROM test").get(); SearchSourceBuilder source = response.source(); assertThat(source.docValueFields(), Matchers.contains( - new DocValueFieldsContext.FieldAndFormat("count", DocValueFieldsContext.USE_DEFAULT_FORMAT))); + new DocValueFieldsContext.FieldAndFormat("count", null))); FetchSourceContext fetchSource = source.fetchSource(); assertThat(fetchSource.includes(), Matchers.arrayContaining("data")); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlTranslateActionIT.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlTranslateActionIT.java index 3dc41ad9dd362..0528b9121d323 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlTranslateActionIT.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlTranslateActionIT.java @@ -35,7 +35,7 @@ public void testSqlTranslateAction() { assertTrue(fetch.fetchSource()); assertArrayEquals(new String[] { "data" }, fetch.includes()); assertEquals( - singletonList(new DocValueFieldsContext.FieldAndFormat("count", DocValueFieldsContext.USE_DEFAULT_FORMAT)), + singletonList(new DocValueFieldsContext.FieldAndFormat("count", null)), source.docValueFields()); assertEquals(singletonList(SortBuilders.fieldSort("count").missing("_last").unmappedType("long")), source.sorts()); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainerTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainerTests.java index 5e675b0be5a72..424964bdbd9f3 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainerTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainerTests.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.xpack.sql.querydsl.container; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.sql.querydsl.query.BoolQuery; import org.elasticsearch.xpack.sql.querydsl.query.MatchAll; @@ -24,7 +23,7 @@ public class QueryContainerTests extends ESTestCase { private Source source = SourceTests.randomSource(); private String path = randomAlphaOfLength(5); private String name = randomAlphaOfLength(5); - private String format = DocValueFieldsContext.USE_DEFAULT_FORMAT; + private String format = null; private boolean hasDocValues = randomBoolean(); public void testRewriteToContainNestedFieldNoQuery() { diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/BoolQueryTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/BoolQueryTests.java index adc733a29c33b..ac6b85538805c 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/BoolQueryTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/BoolQueryTests.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.xpack.sql.querydsl.query; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.search.sort.NestedSortBuilder; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.sql.tree.Source; @@ -53,15 +52,14 @@ public void testContainsNestedField() { public void testAddNestedField() { Query q = boolQueryWithoutNestedChildren(); - assertSame(q, q.addNestedField(randomAlphaOfLength(5), randomAlphaOfLength(5), DocValueFieldsContext.USE_DEFAULT_FORMAT, - randomBoolean())); + assertSame(q, q.addNestedField(randomAlphaOfLength(5), randomAlphaOfLength(5), null, randomBoolean())); String path = randomAlphaOfLength(5); String field = randomAlphaOfLength(5); q = boolQueryWithNestedChildren(path, field); String newField = randomAlphaOfLength(5); boolean hasDocValues = randomBoolean(); - Query rewritten = q.addNestedField(path, newField, DocValueFieldsContext.USE_DEFAULT_FORMAT, hasDocValues); + Query rewritten = q.addNestedField(path, newField, null, hasDocValues); assertNotSame(q, rewritten); assertTrue(rewritten.containsNestedField(path, newField)); } @@ -87,7 +85,7 @@ private Query boolQueryWithoutNestedChildren() { private Query boolQueryWithNestedChildren(String path, String field) { NestedQuery match = new NestedQuery(SourceTests.randomSource(), path, - singletonMap(field, new SimpleImmutableEntry<>(randomBoolean(), DocValueFieldsContext.USE_DEFAULT_FORMAT)), + singletonMap(field, new SimpleImmutableEntry<>(randomBoolean(), null)), new MatchAll(SourceTests.randomSource())); Query matchAll = new MatchAll(SourceTests.randomSource()); Query left; @@ -108,4 +106,4 @@ public void testToString() { new ExistsQuery(new Source(1, 1, StringUtils.EMPTY), "f1"), new ExistsQuery(new Source(1, 7, StringUtils.EMPTY), "f2")).toString()); } -} \ No newline at end of file +} diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/LeafQueryTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/LeafQueryTests.java index 05ef987480910..29ede49d98248 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/LeafQueryTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/LeafQueryTests.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.sql.querydsl.query; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.search.sort.NestedSortBuilder; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.sql.tree.Source; @@ -54,8 +53,7 @@ public void testContainsNestedField() { public void testAddNestedField() { Query query = new DummyLeafQuery(SourceTests.randomSource()); // Leaf queries don't contain nested fields. - assertSame(query, query.addNestedField(randomAlphaOfLength(5), randomAlphaOfLength(5), DocValueFieldsContext.USE_DEFAULT_FORMAT, - randomBoolean())); + assertSame(query, query.addNestedField(randomAlphaOfLength(5), randomAlphaOfLength(5), null, randomBoolean())); } public void testEnrichNestedSort() { diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/NestedQueryTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/NestedQueryTests.java index 5fe69760a694d..818ba04fa1882 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/NestedQueryTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/NestedQueryTests.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.xpack.sql.querydsl.query; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.search.sort.NestedSortBuilder; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; @@ -45,7 +44,7 @@ private static Map> randomFields() { int size = between(0, 5); Map> fields = new HashMap<>(size); while (fields.size() < size) { - fields.put(randomAlphaOfLength(5), new SimpleImmutableEntry<>(randomBoolean(), DocValueFieldsContext.USE_DEFAULT_FORMAT)); + fields.put(randomAlphaOfLength(5), new SimpleImmutableEntry<>(randomBoolean(), null)); } return fields; } @@ -80,18 +79,18 @@ public void testAddNestedField() { NestedQuery q = randomNestedQuery(0); for (String field : q.fields().keySet()) { // add does nothing if the field is already there - assertSame(q, q.addNestedField(q.path(), field, DocValueFieldsContext.USE_DEFAULT_FORMAT, randomBoolean())); + assertSame(q, q.addNestedField(q.path(), field, null, randomBoolean())); String otherPath = randomValueOtherThan(q.path(), () -> randomAlphaOfLength(5)); // add does nothing if the path doesn't match - assertSame(q, q.addNestedField(otherPath, randomAlphaOfLength(5), DocValueFieldsContext.USE_DEFAULT_FORMAT, randomBoolean())); + assertSame(q, q.addNestedField(otherPath, randomAlphaOfLength(5), null, randomBoolean())); } // if the field isn't in the list then add rewrites to a query with all the old fields and the new one String newField = randomValueOtherThanMany(q.fields()::containsKey, () -> randomAlphaOfLength(5)); boolean hasDocValues = randomBoolean(); - NestedQuery added = (NestedQuery) q.addNestedField(q.path(), newField, DocValueFieldsContext.USE_DEFAULT_FORMAT, hasDocValues); + NestedQuery added = (NestedQuery) q.addNestedField(q.path(), newField, null, hasDocValues); assertNotSame(q, added); - assertThat(added.fields(), hasEntry(newField, new SimpleImmutableEntry<>(hasDocValues, DocValueFieldsContext.USE_DEFAULT_FORMAT))); + assertThat(added.fields(), hasEntry(newField, new SimpleImmutableEntry<>(hasDocValues, null))); assertTrue(added.containsNestedField(q.path(), newField)); for (Map.Entry> field : q.fields().entrySet()) { assertThat(added.fields(), hasEntry(field.getKey(), field.getValue())); @@ -133,8 +132,8 @@ public void testEnrichNestedSort() { public void testToString() { NestedQuery q = new NestedQuery(new Source(1, 1, StringUtils.EMPTY), "a.b", - singletonMap("f", new SimpleImmutableEntry<>(true, DocValueFieldsContext.USE_DEFAULT_FORMAT)), + singletonMap("f", new SimpleImmutableEntry<>(true, null)), new MatchAll(new Source(1, 1, StringUtils.EMPTY))); - assertEquals("NestedQuery@1:2[a.b.{f=true=use_field_mapping}[MatchAll@1:2[]]]", q.toString()); + assertEquals("NestedQuery@1:2[a.b.{f=true=null}[MatchAll@1:2[]]]", q.toString()); } -} \ No newline at end of file +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/sql/translate.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/sql/translate.yml index 0a22a189fa7b6..9fa8e6259f5ff 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/sql/translate.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/sql/translate.yml @@ -1,9 +1,8 @@ --- "Translate SQL": - skip: - version: " - 6.3.99" - reason: format option was added in 6.4 - features: warnings + version: " - 6.99.99" + reason: Triggers warnings before 7.0 - do: bulk: @@ -29,7 +28,6 @@ excludes: [] docvalue_fields: - field: int - format: use_field_mapping sort: - int: order: asc From 908c8def063e529ee4f1aa146d43132b9dafd66e Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Wed, 30 Jan 2019 11:34:47 +0200 Subject: [PATCH 042/100] SQL: Skip the nested and object field types in case of an ODBC request (#37948) --- .../plan/logical/command/sys/SysColumns.java | 78 ++++++++++--------- .../logical/command/sys/SysColumnsTests.java | 61 +++++++++++++-- 2 files changed, 97 insertions(+), 42 deletions(-) diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java index 76b58babe2832..b3730ee33405e 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java @@ -15,8 +15,8 @@ import org.elasticsearch.xpack.sql.session.Rows; import org.elasticsearch.xpack.sql.session.SchemaRowSet; import org.elasticsearch.xpack.sql.session.SqlSession; -import org.elasticsearch.xpack.sql.tree.Source; import org.elasticsearch.xpack.sql.tree.NodeInfo; +import org.elasticsearch.xpack.sql.tree.Source; import org.elasticsearch.xpack.sql.type.DataType; import org.elasticsearch.xpack.sql.type.DataTypes; import org.elasticsearch.xpack.sql.type.EsField; @@ -133,42 +133,46 @@ static void fillInRows(String clusterName, String indexName, Map> rows = new ArrayList<>(); SysColumns.fillInRows("test", "index", TypesTests.loadMapping("mapping-multi-field-variation.json", true), null, rows, null, true); - assertEquals(16, rows.size()); + assertEquals(14, rows.size()); assertEquals(24, rows.get(0).size()); List row = rows.get(0); @@ -90,6 +90,16 @@ public void testSysColumnsInOdbcMode() { assertEquals(Short.class, sqlDataType(row).getClass()); assertEquals(Short.class, sqlDataTypeSub(row).getClass()); + row = rows.get(3); + assertEquals("keyword", name(row)); + assertEquals((short) Types.VARCHAR, sqlType(row)); + assertEquals(null, radix(row)); + assertEquals(Integer.MAX_VALUE, bufferLength(row)); + assertNull(decimalPrecision(row)); + assertEquals(Short.class, nullable(row).getClass()); + assertEquals(Short.class, sqlDataType(row).getClass()); + assertEquals(Short.class, sqlDataTypeSub(row).getClass()); + row = rows.get(4); assertEquals("date", name(row)); assertEquals((short) Types.TIMESTAMP, sqlType(row)); @@ -101,17 +111,58 @@ public void testSysColumnsInOdbcMode() { assertEquals(Short.class, sqlDataType(row).getClass()); assertEquals(Short.class, sqlDataTypeSub(row).getClass()); + row = rows.get(5); + assertEquals("unsupported", name(row)); + assertEquals((short) Types.OTHER, sqlType(row)); + assertEquals(null, radix(row)); + assertEquals(0, precision(row)); + assertEquals(0, bufferLength(row)); + assertNull(decimalPrecision(row)); + assertEquals(Short.class, nullable(row).getClass()); + assertEquals(Short.class, sqlDataType(row).getClass()); + assertEquals(Short.class, sqlDataTypeSub(row).getClass()); + + row = rows.get(6); + assertEquals("some.dotted.field", name(row)); + assertEquals((short) Types.VARCHAR, sqlType(row)); + assertEquals(null, radix(row)); + assertEquals(Integer.MAX_VALUE, bufferLength(row)); + assertNull(decimalPrecision(row)); + assertEquals(Short.class, nullable(row).getClass()); + assertEquals(Short.class, sqlDataType(row).getClass()); + assertEquals(Short.class, sqlDataTypeSub(row).getClass()); + row = rows.get(7); - assertEquals("some.dotted", name(row)); - assertEquals((short) Types.STRUCT, sqlType(row)); + assertEquals("some.string", name(row)); + assertEquals((short) Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); - assertEquals(-1, bufferLength(row)); + assertEquals(Integer.MAX_VALUE, bufferLength(row)); assertNull(decimalPrecision(row)); assertEquals(Short.class, nullable(row).getClass()); assertEquals(Short.class, sqlDataType(row).getClass()); assertEquals(Short.class, sqlDataTypeSub(row).getClass()); - row = rows.get(15); + row = rows.get(8); + assertEquals("some.string.normalized", name(row)); + assertEquals((short) Types.VARCHAR, sqlType(row)); + assertEquals(null, radix(row)); + assertEquals(Integer.MAX_VALUE, bufferLength(row)); + assertNull(decimalPrecision(row)); + assertEquals(Short.class, nullable(row).getClass()); + assertEquals(Short.class, sqlDataType(row).getClass()); + assertEquals(Short.class, sqlDataTypeSub(row).getClass()); + + row = rows.get(9); + assertEquals("some.string.typical", name(row)); + assertEquals((short) Types.VARCHAR, sqlType(row)); + assertEquals(null, radix(row)); + assertEquals(Integer.MAX_VALUE, bufferLength(row)); + assertNull(decimalPrecision(row)); + assertEquals(Short.class, nullable(row).getClass()); + assertEquals(Short.class, sqlDataType(row).getClass()); + assertEquals(Short.class, sqlDataTypeSub(row).getClass()); + + row = rows.get(13); assertEquals("some.ambiguous.normalized", name(row)); assertEquals((short) Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); From 9ec4abc31eeaf7f60a4ff3597addcb154b07a42e Mon Sep 17 00:00:00 2001 From: Alexander Reelsen Date: Wed, 30 Jan 2019 10:47:12 +0100 Subject: [PATCH 043/100] Ensure date parsing BWC compatibility (#37929) In order to retain BWC this changes the java date formatters to be able to parse nanoseconds resolution, even if only milliseconds are supported. This used to work on joda time as well so that a user could store a date like `2018-10-03T14:42:44.613469+0000` and then just loose the precision on anything lower than millisecond level. --- .../common/time/DateFormatters.java | 52 +++++++++++-------- .../joda/JavaJodaTimeDuellingTests.java | 26 ++++++++++ 2 files changed, 55 insertions(+), 23 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java b/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java index 2e3c2953ec375..0b92955583f7f 100644 --- a/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java +++ b/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java @@ -108,9 +108,6 @@ public class DateFormatters { .appendLiteral(':') .appendValue(SECOND_OF_MINUTE, 2, 2, SignStyle.NOT_NEGATIVE) .optionalStart() - .appendFraction(NANO_OF_SECOND, 3, 3, true) - .optionalEnd() - .optionalStart() .appendFraction(NANO_OF_SECOND, 3, 9, true) .optionalEnd() .optionalEnd() @@ -205,7 +202,7 @@ public class DateFormatters { .appendValue(HOUR_OF_DAY, 2, 2, SignStyle.NOT_NEGATIVE) .appendValue(MINUTE_OF_HOUR, 2, 2, SignStyle.NOT_NEGATIVE) .appendValue(SECOND_OF_MINUTE, 2, 2, SignStyle.NOT_NEGATIVE) - .appendFraction(NANO_OF_SECOND, 1, 3, true) + .appendFraction(NANO_OF_SECOND, 1, 9, true) .toFormatter(Locale.ROOT); private static final DateTimeFormatter BASIC_TIME_PRINTER = new DateTimeFormatterBuilder() @@ -311,7 +308,7 @@ public class DateFormatters { private static final DateFormatter BASIC_ORDINAL_DATE_TIME = new JavaDateFormatter("basic_ordinal_date_time", new DateTimeFormatterBuilder().appendPattern("yyyyDDD").append(BASIC_T_TIME_PRINTER) .appendZoneOrOffsetId().toFormatter(Locale.ROOT), - new DateTimeFormatterBuilder().appendPattern("yyyyDDD").append(BASIC_T_TIME_PRINTER) + new DateTimeFormatterBuilder().appendPattern("yyyyDDD").append(BASIC_T_TIME_FORMATTER) .appendZoneOrOffsetId().toFormatter(Locale.ROOT), new DateTimeFormatterBuilder().appendPattern("yyyyDDD").append(BASIC_T_TIME_FORMATTER) .append(TIME_ZONE_FORMATTER_NO_COLON).toFormatter(Locale.ROOT) @@ -419,7 +416,7 @@ public class DateFormatters { .appendValue(HOUR_OF_DAY, 2, 2, SignStyle.NOT_NEGATIVE) .appendValue(MINUTE_OF_HOUR, 2, 2, SignStyle.NOT_NEGATIVE) .appendValue(SECOND_OF_MINUTE, 2, 2, SignStyle.NOT_NEGATIVE) - .appendFraction(NANO_OF_SECOND, 3, 3, true) + .appendFraction(NANO_OF_SECOND, 3, 9, true) .appendZoneOrOffsetId() .toFormatter(Locale.ROOT), new DateTimeFormatterBuilder() @@ -428,7 +425,7 @@ public class DateFormatters { .appendValue(HOUR_OF_DAY, 2, 2, SignStyle.NOT_NEGATIVE) .appendValue(MINUTE_OF_HOUR, 2, 2, SignStyle.NOT_NEGATIVE) .appendValue(SECOND_OF_MINUTE, 2, 2, SignStyle.NOT_NEGATIVE) - .appendFraction(NANO_OF_SECOND, 3, 3, true) + .appendFraction(NANO_OF_SECOND, 3, 9, true) .append(TIME_ZONE_FORMATTER_NO_COLON) .toFormatter(Locale.ROOT) ); @@ -485,7 +482,7 @@ public class DateFormatters { .appendLiteral('T') .append(STRICT_HOUR_MINUTE_SECOND_FORMATTER) .optionalStart() - .appendFraction(NANO_OF_SECOND, 3, 3, true) + .appendFraction(NANO_OF_SECOND, 3, 9, true) .optionalEnd() .toFormatter(Locale.ROOT); @@ -542,7 +539,7 @@ public class DateFormatters { // NOTE: this is not a strict formatter to retain the joda time based behaviour, even though it's named like this private static final DateTimeFormatter STRICT_HOUR_MINUTE_SECOND_MILLIS_FORMATTER = new DateTimeFormatterBuilder() .append(STRICT_HOUR_MINUTE_SECOND_FORMATTER) - .appendFraction(NANO_OF_SECOND, 1, 3, true) + .appendFraction(NANO_OF_SECOND, 1, 9, true) .toFormatter(Locale.ROOT); private static final DateTimeFormatter STRICT_HOUR_MINUTE_SECOND_MILLIS_PRINTER = new DateTimeFormatterBuilder() @@ -582,8 +579,8 @@ public class DateFormatters { .append(STRICT_YEAR_MONTH_DAY_FORMATTER) .appendLiteral("T") .append(STRICT_HOUR_MINUTE_SECOND_FORMATTER) - // this one here is lenient as well to retain joda time based bwc compatibility - .appendFraction(NANO_OF_SECOND, 1, 3, true) + // this one here is lenient as well to retain joda time based bwc compatibility + .appendFraction(NANO_OF_SECOND, 1, 9, true) .toFormatter(Locale.ROOT) ); @@ -599,7 +596,7 @@ public class DateFormatters { .appendLiteral("T") .append(STRICT_HOUR_MINUTE_SECOND_FORMATTER) // this one here is lenient as well to retain joda time based bwc compatibility - .appendFraction(NANO_OF_SECOND, 1, 3, true) + .appendFraction(NANO_OF_SECOND, 1, 9, true) .toFormatter(Locale.ROOT) ); @@ -625,7 +622,7 @@ public class DateFormatters { .optionalStart() .appendLiteral(':') .appendValue(SECOND_OF_MINUTE, 2, 2, SignStyle.NOT_NEGATIVE) - .appendFraction(NANO_OF_SECOND, 3, 3, true) + .appendFraction(NANO_OF_SECOND, 3, 9, true) .optionalEnd() .toFormatter(Locale.ROOT); @@ -649,7 +646,7 @@ public class DateFormatters { .appendValue(MINUTE_OF_HOUR, 2, 2, SignStyle.NOT_NEGATIVE) .appendLiteral(':') .appendValue(SECOND_OF_MINUTE, 2, 2, SignStyle.NOT_NEGATIVE) - .appendFraction(NANO_OF_SECOND, 1, 3, true) + .appendFraction(NANO_OF_SECOND, 1, 9, true) .toFormatter(Locale.ROOT); private static final DateTimeFormatter STRICT_TIME_PRINTER = new DateTimeFormatterBuilder() @@ -880,7 +877,7 @@ public class DateFormatters { .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE) .optionalEnd() .optionalStart() - .appendFraction(NANO_OF_SECOND, 1, 3, true) + .appendFraction(NANO_OF_SECOND, 1, 9, true) .optionalEnd() .optionalStart().appendZoneOrOffsetId().optionalEnd() .optionalStart().appendOffset("+HHmm", "Z").optionalEnd() @@ -904,6 +901,15 @@ public class DateFormatters { .appendFraction(NANO_OF_SECOND, 1, 3, true) .toFormatter(Locale.ROOT); + private static final DateTimeFormatter HOUR_MINUTE_SECOND_FRACTION_FORMATTER = new DateTimeFormatterBuilder() + .appendValue(HOUR_OF_DAY, 1, 2, SignStyle.NOT_NEGATIVE) + .appendLiteral(':') + .appendValue(MINUTE_OF_HOUR, 1, 2, SignStyle.NOT_NEGATIVE) + .appendLiteral(':') + .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE) + .appendFraction(NANO_OF_SECOND, 1, 9, true) + .toFormatter(Locale.ROOT); + private static final DateTimeFormatter ORDINAL_DATE_FORMATTER = new DateTimeFormatterBuilder() .appendValue(ChronoField.YEAR, 4, 10, SignStyle.EXCEEDS_PAD) .appendLiteral('-') @@ -936,7 +942,7 @@ public class DateFormatters { private static final DateTimeFormatter TIME_PREFIX = new DateTimeFormatterBuilder() .append(TIME_NO_MILLIS_FORMATTER) - .appendFraction(NANO_OF_SECOND, 1, 3, true) + .appendFraction(NANO_OF_SECOND, 1, 9, true) .toFormatter(Locale.ROOT); private static final DateTimeFormatter WEEK_DATE_FORMATTER = new DateTimeFormatterBuilder() @@ -974,8 +980,7 @@ public class DateFormatters { /* * Returns a formatter that combines a full date, two digit hour of day, * two digit minute of hour, two digit second of minute, and three digit - * fraction of second (yyyy-MM-dd'T'HH:mm:ss.SSS). Parsing will parse up - * to 3 fractional second digits. + * fraction of second (yyyy-MM-dd'T'HH:mm:ss.SSS). */ private static final DateFormatter DATE_HOUR_MINUTE_SECOND_MILLIS = new JavaDateFormatter("date_hour_minute_second_millis", @@ -990,7 +995,8 @@ public class DateFormatters { .append(HOUR_MINUTE_SECOND_MILLIS_FORMATTER) .toFormatter(Locale.ROOT)); - private static final DateFormatter DATE_HOUR_MINUTE_SECOND_FRACTION = new JavaDateFormatter("date_hour_minute_second_fraction", + private static final DateFormatter DATE_HOUR_MINUTE_SECOND_FRACTION = + new JavaDateFormatter("date_hour_minute_second_fraction", new DateTimeFormatterBuilder() .append(STRICT_YEAR_MONTH_DAY_FORMATTER) .appendLiteral("T") @@ -999,7 +1005,7 @@ public class DateFormatters { new DateTimeFormatterBuilder() .append(DATE_FORMATTER) .appendLiteral("T") - .append(HOUR_MINUTE_SECOND_MILLIS_FORMATTER) + .append(HOUR_MINUTE_SECOND_FRACTION_FORMATTER) .toFormatter(Locale.ROOT)); /* @@ -1034,7 +1040,7 @@ public class DateFormatters { .optionalStart() .appendLiteral(':') .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE) - .appendFraction(NANO_OF_SECOND, 1, 3, true) + .appendFraction(NANO_OF_SECOND, 1, 9, true) .optionalEnd() .toFormatter(Locale.ROOT); @@ -1106,7 +1112,7 @@ public class DateFormatters { STRICT_HOUR_MINUTE_SECOND_MILLIS_PRINTER, HOUR_MINUTE_SECOND_MILLIS_FORMATTER); private static final DateFormatter HOUR_MINUTE_SECOND_FRACTION = new JavaDateFormatter("hour_minute_second_fraction", - STRICT_HOUR_MINUTE_SECOND_MILLIS_PRINTER, HOUR_MINUTE_SECOND_MILLIS_FORMATTER); + STRICT_HOUR_MINUTE_SECOND_MILLIS_PRINTER, HOUR_MINUTE_SECOND_FRACTION_FORMATTER); /* * Returns a formatter for a two digit hour of day and two digit minute of @@ -1142,7 +1148,7 @@ public class DateFormatters { .optionalStart() .appendLiteral(':') .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE) - .appendFraction(NANO_OF_SECOND, 1, 3, true) + .appendFraction(NANO_OF_SECOND, 1, 9, true) .optionalEnd() .toFormatter(Locale.ROOT); diff --git a/server/src/test/java/org/elasticsearch/common/joda/JavaJodaTimeDuellingTests.java b/server/src/test/java/org/elasticsearch/common/joda/JavaJodaTimeDuellingTests.java index c7abea63be081..423592d6d18d7 100644 --- a/server/src/test/java/org/elasticsearch/common/joda/JavaJodaTimeDuellingTests.java +++ b/server/src/test/java/org/elasticsearch/common/joda/JavaJodaTimeDuellingTests.java @@ -97,18 +97,21 @@ public void testDuellingFormatsValidParsing() { assertSameDate("20181126T121212+0100", "basic_date_time_no_millis"); assertSameDate("2018363", "basic_ordinal_date"); assertSameDate("2018363T121212.123Z", "basic_ordinal_date_time"); + assertSameDate("2018363T121212.123456789Z", "basic_ordinal_date_time"); assertSameDate("2018363T121212.123+0100", "basic_ordinal_date_time"); assertSameDate("2018363T121212.123+01:00", "basic_ordinal_date_time"); assertSameDate("2018363T121212Z", "basic_ordinal_date_time_no_millis"); assertSameDate("2018363T121212+0100", "basic_ordinal_date_time_no_millis"); assertSameDate("2018363T121212+01:00", "basic_ordinal_date_time_no_millis"); assertSameDate("121212.123Z", "basic_time"); + assertSameDate("121212.123456789Z", "basic_time"); assertSameDate("121212.123+0100", "basic_time"); assertSameDate("121212.123+01:00", "basic_time"); assertSameDate("121212Z", "basic_time_no_millis"); assertSameDate("121212+0100", "basic_time_no_millis"); assertSameDate("121212+01:00", "basic_time_no_millis"); assertSameDate("T121212.123Z", "basic_t_time"); + assertSameDate("T121212.123456789Z", "basic_t_time"); assertSameDate("T121212.123+0100", "basic_t_time"); assertSameDate("T121212.123+01:00", "basic_t_time"); assertSameDate("T121212Z", "basic_t_time_no_millis"); @@ -118,6 +121,7 @@ public void testDuellingFormatsValidParsing() { assertSameDate("1W313", "basic_week_date"); assertSameDate("18W313", "basic_week_date"); assertSameDate("2018W313T121212.123Z", "basic_week_date_time"); + assertSameDate("2018W313T121212.123456789Z", "basic_week_date_time"); assertSameDate("2018W313T121212.123+0100", "basic_week_date_time"); assertSameDate("2018W313T121212.123+01:00", "basic_week_date_time"); assertSameDate("2018W313T121212Z", "basic_week_date_time_no_millis"); @@ -138,7 +142,9 @@ public void testDuellingFormatsValidParsing() { assertSameDate("2018-12-31T12:12:1", "date_hour_minute_second"); assertSameDate("2018-12-31T12:12:12.123", "date_hour_minute_second_fraction"); + assertSameDate("2018-12-31T12:12:12.123456789", "date_hour_minute_second_fraction"); assertSameDate("2018-12-31T12:12:12.123", "date_hour_minute_second_millis"); + assertParseException("2018-12-31T12:12:12.123456789", "date_hour_minute_second_millis"); assertSameDate("2018-12-31T12:12:12.1", "date_hour_minute_second_millis"); assertSameDate("2018-12-31T12:12:12.1", "date_hour_minute_second_fraction"); @@ -148,7 +154,9 @@ public void testDuellingFormatsValidParsing() { assertSameDate("2018-05-30T20:21", "date_optional_time"); assertSameDate("2018-05-30T20:21:23", "date_optional_time"); assertSameDate("2018-05-30T20:21:23.123", "date_optional_time"); + assertSameDate("2018-05-30T20:21:23.123456789", "date_optional_time"); assertSameDate("2018-05-30T20:21:23.123Z", "date_optional_time"); + assertSameDate("2018-05-30T20:21:23.123456789Z", "date_optional_time"); assertSameDate("2018-05-30T20:21:23.123+0100", "date_optional_time"); assertSameDate("2018-05-30T20:21:23.123+01:00", "date_optional_time"); assertSameDate("2018-12-1", "date_optional_time"); @@ -158,12 +166,14 @@ public void testDuellingFormatsValidParsing() { assertSameDate("2018-12-31T1:15:30", "date_optional_time"); assertSameDate("2018-12-31T10:15:30.123Z", "date_time"); + assertSameDate("2018-12-31T10:15:30.123456789Z", "date_time"); assertSameDate("2018-12-31T10:15:30.123+0100", "date_time"); assertSameDate("2018-12-31T10:15:30.123+01:00", "date_time"); assertSameDate("2018-12-31T10:15:30.11Z", "date_time"); assertSameDate("2018-12-31T10:15:30.11+0100", "date_time"); assertSameDate("2018-12-31T10:15:30.11+01:00", "date_time"); assertSameDate("2018-12-31T10:15:3.123Z", "date_time"); + assertSameDate("2018-12-31T10:15:3.123456789Z", "date_time"); assertSameDate("2018-12-31T10:15:3.123+0100", "date_time"); assertSameDate("2018-12-31T10:15:3.123+01:00", "date_time"); @@ -193,9 +203,11 @@ public void testDuellingFormatsValidParsing() { assertSameDate("12:12:1", "hour_minute_second"); assertSameDate("12:12:12.123", "hour_minute_second_fraction"); + assertSameDate("12:12:12.123456789", "hour_minute_second_fraction"); assertSameDate("12:12:12.1", "hour_minute_second_fraction"); assertParseException("12:12:12", "hour_minute_second_fraction"); assertSameDate("12:12:12.123", "hour_minute_second_millis"); + assertParseException("12:12:12.123456789", "hour_minute_second_millis"); assertSameDate("12:12:12.1", "hour_minute_second_millis"); assertParseException("12:12:12", "hour_minute_second_millis"); @@ -203,9 +215,11 @@ public void testDuellingFormatsValidParsing() { assertSameDate("2018-1", "ordinal_date"); assertSameDate("2018-128T10:15:30.123Z", "ordinal_date_time"); + assertSameDate("2018-128T10:15:30.123456789Z", "ordinal_date_time"); assertSameDate("2018-128T10:15:30.123+0100", "ordinal_date_time"); assertSameDate("2018-128T10:15:30.123+01:00", "ordinal_date_time"); assertSameDate("2018-1T10:15:30.123Z", "ordinal_date_time"); + assertSameDate("2018-1T10:15:30.123456789Z", "ordinal_date_time"); assertSameDate("2018-1T10:15:30.123+0100", "ordinal_date_time"); assertSameDate("2018-1T10:15:30.123+01:00", "ordinal_date_time"); @@ -217,6 +231,7 @@ public void testDuellingFormatsValidParsing() { assertSameDate("2018-1T10:15:30+01:00", "ordinal_date_time_no_millis"); assertSameDate("10:15:30.123Z", "time"); + assertSameDate("10:15:30.123456789Z", "time"); assertSameDate("10:15:30.123+0100", "time"); assertSameDate("10:15:30.123+01:00", "time"); assertSameDate("1:15:30.123Z", "time"); @@ -249,6 +264,7 @@ public void testDuellingFormatsValidParsing() { assertParseException("10:15:3", "time_no_millis"); assertSameDate("T10:15:30.123Z", "t_time"); + assertSameDate("T10:15:30.123456789Z", "t_time"); assertSameDate("T10:15:30.123+0100", "t_time"); assertSameDate("T10:15:30.123+01:00", "t_time"); assertSameDate("T1:15:30.123Z", "t_time"); @@ -286,6 +302,7 @@ public void testDuellingFormatsValidParsing() { assertJavaTimeParseException("2012-W1-8", "week_date"); assertSameDate("2012-W48-6T10:15:30.123Z", "week_date_time"); + assertSameDate("2012-W48-6T10:15:30.123456789Z", "week_date_time"); assertSameDate("2012-W48-6T10:15:30.123+0100", "week_date_time"); assertSameDate("2012-W48-6T10:15:30.123+01:00", "week_date_time"); assertSameDate("2012-W1-6T10:15:30.123Z", "week_date_time"); @@ -326,9 +343,11 @@ public void testDuelingStrictParsing() { assertSameDate("2018W313", "strict_basic_week_date"); assertParseException("18W313", "strict_basic_week_date"); assertSameDate("2018W313T121212.123Z", "strict_basic_week_date_time"); + assertSameDate("2018W313T121212.123456789Z", "strict_basic_week_date_time"); assertSameDate("2018W313T121212.123+0100", "strict_basic_week_date_time"); assertSameDate("2018W313T121212.123+01:00", "strict_basic_week_date_time"); assertParseException("2018W313T12128.123Z", "strict_basic_week_date_time"); + assertParseException("2018W313T12128.123456789Z", "strict_basic_week_date_time"); assertParseException("2018W313T81212.123Z", "strict_basic_week_date_time"); assertParseException("2018W313T12812.123Z", "strict_basic_week_date_time"); assertParseException("2018W313T12812.1Z", "strict_basic_week_date_time"); @@ -354,6 +373,7 @@ public void testDuelingStrictParsing() { assertSameDate("2018-12-31T12:12:12", "strict_date_hour_minute_second"); assertParseException("2018-12-31T12:12:1", "strict_date_hour_minute_second"); assertSameDate("2018-12-31T12:12:12.123", "strict_date_hour_minute_second_fraction"); + assertSameDate("2018-12-31T12:12:12.123456789", "strict_date_hour_minute_second_fraction"); assertSameDate("2018-12-31T12:12:12.123", "strict_date_hour_minute_second_millis"); assertSameDate("2018-12-31T12:12:12.1", "strict_date_hour_minute_second_millis"); assertSameDate("2018-12-31T12:12:12.1", "strict_date_hour_minute_second_fraction"); @@ -373,6 +393,7 @@ public void testDuelingStrictParsing() { assertParseException("2018-12-31T9:15:30", "strict_date_optional_time"); assertSameDate("2015-01-04T00:00Z", "strict_date_optional_time"); assertSameDate("2018-12-31T10:15:30.123Z", "strict_date_time"); + assertSameDate("2018-12-31T10:15:30.123456789Z", "strict_date_time"); assertSameDate("2018-12-31T10:15:30.123+0100", "strict_date_time"); assertSameDate("2018-12-31T10:15:30.123+01:00", "strict_date_time"); assertSameDate("2018-12-31T10:15:30.11Z", "strict_date_time"); @@ -397,6 +418,7 @@ public void testDuelingStrictParsing() { assertSameDate("12:12:01", "strict_hour_minute_second"); assertParseException("12:12:1", "strict_hour_minute_second"); assertSameDate("12:12:12.123", "strict_hour_minute_second_fraction"); + assertSameDate("12:12:12.123456789", "strict_hour_minute_second_fraction"); assertSameDate("12:12:12.1", "strict_hour_minute_second_fraction"); assertParseException("12:12:12", "strict_hour_minute_second_fraction"); assertSameDate("12:12:12.123", "strict_hour_minute_second_millis"); @@ -406,6 +428,7 @@ public void testDuelingStrictParsing() { assertParseException("2018-1", "strict_ordinal_date"); assertSameDate("2018-128T10:15:30.123Z", "strict_ordinal_date_time"); + assertSameDate("2018-128T10:15:30.123456789Z", "strict_ordinal_date_time"); assertSameDate("2018-128T10:15:30.123+0100", "strict_ordinal_date_time"); assertSameDate("2018-128T10:15:30.123+01:00", "strict_ordinal_date_time"); assertParseException("2018-1T10:15:30.123Z", "strict_ordinal_date_time"); @@ -416,6 +439,7 @@ public void testDuelingStrictParsing() { assertParseException("2018-1T10:15:30Z", "strict_ordinal_date_time_no_millis"); assertSameDate("10:15:30.123Z", "strict_time"); + assertSameDate("10:15:30.123456789Z", "strict_time"); assertSameDate("10:15:30.123+0100", "strict_time"); assertSameDate("10:15:30.123+01:00", "strict_time"); assertParseException("1:15:30.123Z", "strict_time"); @@ -436,6 +460,7 @@ public void testDuelingStrictParsing() { assertParseException("10:15:3", "strict_time_no_millis"); assertSameDate("T10:15:30.123Z", "strict_t_time"); + assertSameDate("T10:15:30.123456789Z", "strict_t_time"); assertSameDate("T10:15:30.123+0100", "strict_t_time"); assertSameDate("T10:15:30.123+01:00", "strict_t_time"); assertParseException("T1:15:30.123Z", "strict_t_time"); @@ -466,6 +491,7 @@ public void testDuelingStrictParsing() { assertJavaTimeParseException("2012-W01-8", "strict_week_date"); assertSameDate("2012-W48-6T10:15:30.123Z", "strict_week_date_time"); + assertSameDate("2012-W48-6T10:15:30.123456789Z", "strict_week_date_time"); assertSameDate("2012-W48-6T10:15:30.123+0100", "strict_week_date_time"); assertSameDate("2012-W48-6T10:15:30.123+01:00", "strict_week_date_time"); assertParseException("2012-W1-6T10:15:30.123Z", "strict_week_date_time"); From 7205833f92df93bfdb09296aafade865c57af0d4 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 30 Jan 2019 10:43:57 +0100 Subject: [PATCH 044/100] Revert "Documented default values for index follow request parameters. (#37917)" This reverts commit 4da7a446489674beff665a3093037fdb25f93787. --- .../ccr/apis/follow-request-body.asciidoc | 62 +------------------ 1 file changed, 1 insertion(+), 61 deletions(-) diff --git a/docs/reference/ccr/apis/follow-request-body.asciidoc b/docs/reference/ccr/apis/follow-request-body.asciidoc index ec384deee8b81..7215cc01302a1 100644 --- a/docs/reference/ccr/apis/follow-request-body.asciidoc +++ b/docs/reference/ccr/apis/follow-request-body.asciidoc @@ -1,5 +1,3 @@ -[role="xpack"] -[testenv="platinum"] `max_read_request_operation_count`:: (integer) the maximum number of operations to pull per read from the remote cluster @@ -43,62 +41,4 @@ remote cluster when the follower index is synchronized with the leader index; when the timeout has elapsed, the poll for operations will return to the follower so that it can update some statistics, and then the follower will - immediately attempt to read from the leader again - -===== Default values - -////////////////////////// - -[source,js] --------------------------------------------------- -PUT /follower_index/_ccr/follow -{ - "remote_cluster" : "remote_cluster", - "leader_index" : "leader_index" -} --------------------------------------------------- -// CONSOLE -// TESTSETUP -// TEST[setup:remote_cluster_and_leader_index] - -[source,js] --------------------------------------------------- -POST /follower_index/_ccr/pause_follow --------------------------------------------------- -// CONSOLE -// TEARDOWN - -[source,js] --------------------------------------------------- -GET /follower_index/_ccr/info?filter_path=follower_indices.parameters --------------------------------------------------- -// CONSOLE - -////////////////////////// - -The following output from the follow info api describes all the default -values for the above described index follow request parameters: - -[source,js] --------------------------------------------------- -{ - "follower_indices" : [ - { - "parameters" : { - "max_read_request_operation_count" : 5120, - "max_read_request_size" : "32mb", - "max_outstanding_read_requests" : 12, - "max_write_request_operation_count" : 5120, - "max_write_request_size" : "9223372036854775807b", - "max_outstanding_write_requests" : 9, - "max_write_buffer_count" : 2147483647, - "max_write_buffer_size" : "512mb", - "max_retry_delay" : "500ms", - "read_poll_timeout" : "1m" - } - } - ] -} - --------------------------------------------------- -// TESTRESPONSE \ No newline at end of file + immediately attempt to read from the leader again \ No newline at end of file From e959dbaa998482bee009f3064b6e3edea60301e5 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 30 Jan 2019 10:49:42 +0100 Subject: [PATCH 045/100] Revert "Revert "Documented default values for index follow request parameters. (#37917)"" This reverts commit 7205833f92df93bfdb09296aafade865c57af0d4. --- .../ccr/apis/follow-request-body.asciidoc | 62 ++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/docs/reference/ccr/apis/follow-request-body.asciidoc b/docs/reference/ccr/apis/follow-request-body.asciidoc index 7215cc01302a1..ec384deee8b81 100644 --- a/docs/reference/ccr/apis/follow-request-body.asciidoc +++ b/docs/reference/ccr/apis/follow-request-body.asciidoc @@ -1,3 +1,5 @@ +[role="xpack"] +[testenv="platinum"] `max_read_request_operation_count`:: (integer) the maximum number of operations to pull per read from the remote cluster @@ -41,4 +43,62 @@ remote cluster when the follower index is synchronized with the leader index; when the timeout has elapsed, the poll for operations will return to the follower so that it can update some statistics, and then the follower will - immediately attempt to read from the leader again \ No newline at end of file + immediately attempt to read from the leader again + +===== Default values + +////////////////////////// + +[source,js] +-------------------------------------------------- +PUT /follower_index/_ccr/follow +{ + "remote_cluster" : "remote_cluster", + "leader_index" : "leader_index" +} +-------------------------------------------------- +// CONSOLE +// TESTSETUP +// TEST[setup:remote_cluster_and_leader_index] + +[source,js] +-------------------------------------------------- +POST /follower_index/_ccr/pause_follow +-------------------------------------------------- +// CONSOLE +// TEARDOWN + +[source,js] +-------------------------------------------------- +GET /follower_index/_ccr/info?filter_path=follower_indices.parameters +-------------------------------------------------- +// CONSOLE + +////////////////////////// + +The following output from the follow info api describes all the default +values for the above described index follow request parameters: + +[source,js] +-------------------------------------------------- +{ + "follower_indices" : [ + { + "parameters" : { + "max_read_request_operation_count" : 5120, + "max_read_request_size" : "32mb", + "max_outstanding_read_requests" : 12, + "max_write_request_operation_count" : 5120, + "max_write_request_size" : "9223372036854775807b", + "max_outstanding_write_requests" : 9, + "max_write_buffer_count" : 2147483647, + "max_write_buffer_size" : "512mb", + "max_retry_delay" : "500ms", + "read_poll_timeout" : "1m" + } + } + ] +} + +-------------------------------------------------- +// TESTRESPONSE \ No newline at end of file From 3865435a01a5b60465f37f036953c306e950a75b Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 30 Jan 2019 11:02:16 +0100 Subject: [PATCH 046/100] Docs test fix, wait for shards active. (a restore needs to be complete, which happens in the background and by default the ccr put follow api doesn't wait for this) (this was a recent change and the pr that added this docs test, did not include this change) Relates to #37917 --- docs/reference/ccr/apis/follow-request-body.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/ccr/apis/follow-request-body.asciidoc b/docs/reference/ccr/apis/follow-request-body.asciidoc index ec384deee8b81..e7e6ae2e26a05 100644 --- a/docs/reference/ccr/apis/follow-request-body.asciidoc +++ b/docs/reference/ccr/apis/follow-request-body.asciidoc @@ -51,7 +51,7 @@ [source,js] -------------------------------------------------- -PUT /follower_index/_ccr/follow +PUT /follower_index/_ccr/follow?wait_for_active_shards=1 { "remote_cluster" : "remote_cluster", "leader_index" : "leader_index" From ba285a56a72655d189ee11aeed8c9eb8d2043ff9 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 30 Jan 2019 05:25:17 -0500 Subject: [PATCH 047/100] Fix limit on retaining sequence number (#37992) We only assign non-negative sequence numbers to operations, so the lower limit on retaining sequence numbers should be that it is non-negative only. --- .../java/org/elasticsearch/index/seqno/RetentionLease.java | 2 +- .../elasticsearch/index/engine/InternalEngineTests.java | 2 +- .../elasticsearch/index/engine/SoftDeletesPolicyTests.java | 2 +- .../index/seqno/ReplicationTrackerRetentionLeaseTests.java | 2 +- .../elasticsearch/index/seqno/RetentionLeaseSyncIT.java | 4 ++-- .../org/elasticsearch/index/seqno/RetentionLeaseTests.java | 7 +++---- .../index/shard/IndexShardRetentionLeaseTests.java | 2 +- 7 files changed, 10 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLease.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLease.java index 13e3381b11553..24d144d810d9c 100644 --- a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLease.java +++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLease.java @@ -101,7 +101,7 @@ public RetentionLease(final String id, final long retainingSequenceNumber, final // retention lease IDs can not contain these characters because they are used in encoding retention leases throw new IllegalArgumentException("retention lease ID can not contain any of [:;,] but was [" + id + "]"); } - if (retainingSequenceNumber < SequenceNumbers.UNASSIGNED_SEQ_NO) { + if (retainingSequenceNumber < 0) { throw new IllegalArgumentException("retention lease retaining sequence number [" + retainingSequenceNumber + "] out of range"); } if (timestamp < 0) { diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index f57bff72fc57c..d984d1702f257 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -5313,7 +5313,7 @@ public void testKeepMinRetainedSeqNoByMergePolicy() throws IOException { final List leases = new ArrayList<>(length); for (int i = 0; i < length; i++) { final String id = randomAlphaOfLength(8); - final long retainingSequenceNumber = randomLongBetween(0L, Math.max(0L, globalCheckpoint.get())); + final long retainingSequenceNumber = randomLongBetween(0, Math.max(0, globalCheckpoint.get())); final long timestamp = randomLongBetween(0L, Long.MAX_VALUE); final String source = randomAlphaOfLength(8); leases.add(new RetentionLease(id, retainingSequenceNumber, timestamp, source)); diff --git a/server/src/test/java/org/elasticsearch/index/engine/SoftDeletesPolicyTests.java b/server/src/test/java/org/elasticsearch/index/engine/SoftDeletesPolicyTests.java index 310e83e9d2cef..e15372d687e55 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/SoftDeletesPolicyTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/SoftDeletesPolicyTests.java @@ -49,7 +49,7 @@ public void testSoftDeletesRetentionLock() { AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); final AtomicLong[] retainingSequenceNumbers = new AtomicLong[randomIntBetween(0, 8)]; for (int i = 0; i < retainingSequenceNumbers.length; i++) { - retainingSequenceNumbers[i] = new AtomicLong(SequenceNumbers.UNASSIGNED_SEQ_NO); + retainingSequenceNumbers[i] = new AtomicLong(); } final Supplier> retentionLeasesSupplier = () -> { diff --git a/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java b/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java index 7a867027412e1..90eb162374469 100644 --- a/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java +++ b/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java @@ -156,7 +156,7 @@ private void runExpirationTest(final boolean primaryMode) { replicationTracker.activatePrimaryMode(SequenceNumbers.NO_OPS_PERFORMED); } final long[] retainingSequenceNumbers = new long[1]; - retainingSequenceNumbers[0] = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE); + retainingSequenceNumbers[0] = randomLongBetween(0, Long.MAX_VALUE); if (primaryMode) { replicationTracker.addRetentionLease("0", retainingSequenceNumbers[0], "test-0", ActionListener.wrap(() -> {})); } else { diff --git a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncIT.java b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncIT.java index 7d6e5fa2dc5a6..a99d0caea8e6c 100644 --- a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncIT.java +++ b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncIT.java @@ -67,7 +67,7 @@ public void testRetentionLeasesSyncedOnAdd() throws Exception { final Map currentRetentionLeases = new HashMap<>(); for (int i = 0; i < length; i++) { final String id = randomValueOtherThanMany(currentRetentionLeases.keySet()::contains, () -> randomAlphaOfLength(8)); - final long retainingSequenceNumber = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE); + final long retainingSequenceNumber = randomLongBetween(0, Long.MAX_VALUE); final String source = randomAlphaOfLength(8); final CountDownLatch latch = new CountDownLatch(1); final ActionListener listener = ActionListener.wrap(r -> latch.countDown(), e -> fail(e.toString())); @@ -119,7 +119,7 @@ public void testRetentionLeasesSyncOnExpiration() throws Exception { final int length = randomIntBetween(1, 8); for (int i = 0; i < length; i++) { final String id = randomAlphaOfLength(8); - final long retainingSequenceNumber = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE); + final long retainingSequenceNumber = randomLongBetween(0, Long.MAX_VALUE); final String source = randomAlphaOfLength(8); final CountDownLatch latch = new CountDownLatch(1); final ActionListener listener = ActionListener.wrap(r -> latch.countDown(), e -> fail(e.toString())); diff --git a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseTests.java b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseTests.java index 500393f2cfac2..1a8d159c18757 100644 --- a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseTests.java +++ b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseTests.java @@ -28,7 +28,6 @@ import java.util.Collection; import java.util.List; -import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; @@ -53,7 +52,7 @@ public void testEmptyId() { } public void testRetainingSequenceNumberOutOfRange() { - final long retainingSequenceNumber = randomLongBetween(Long.MIN_VALUE, UNASSIGNED_SEQ_NO - 1); + final long retainingSequenceNumber = randomLongBetween(Long.MIN_VALUE, -1); final IllegalArgumentException e = expectThrows( IllegalArgumentException.class, () -> new RetentionLease("id", retainingSequenceNumber, randomNonNegativeLong(), "source")); @@ -66,7 +65,7 @@ public void testTimestampOutOfRange() { final long timestamp = randomLongBetween(Long.MIN_VALUE, -1); final IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> new RetentionLease("id", randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE), timestamp, "source")); + () -> new RetentionLease("id", randomNonNegativeLong(), timestamp, "source")); assertThat(e, hasToString(containsString("retention lease timestamp [" + timestamp + "] out of range"))); } @@ -87,7 +86,7 @@ public void testEmptySource() { public void testRetentionLeaseSerialization() throws IOException { final String id = randomAlphaOfLength(8); - final long retainingSequenceNumber = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE); + final long retainingSequenceNumber = randomLongBetween(0, Long.MAX_VALUE); final long timestamp = randomNonNegativeLong(); final String source = randomAlphaOfLength(8); final RetentionLease retentionLease = new RetentionLease(id, retainingSequenceNumber, timestamp, source); diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardRetentionLeaseTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardRetentionLeaseTests.java index cd7d2a2c12cb8..26e67fb6dd264 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardRetentionLeaseTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardRetentionLeaseTests.java @@ -113,7 +113,7 @@ private void runExpirationTest(final boolean primary) throws IOException { final IndexShard indexShard = newStartedShard(primary, settings, new InternalEngineFactory()); try { final long[] retainingSequenceNumbers = new long[1]; - retainingSequenceNumbers[0] = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE); + retainingSequenceNumbers[0] = randomLongBetween(0, Long.MAX_VALUE); if (primary) { indexShard.addRetentionLease("0", retainingSequenceNumbers[0], "test-0", ActionListener.wrap(() -> {})); } else { From b91d587275cbce37142d46d0f2ef95cdc94ae87e Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Wed, 30 Jan 2019 12:05:54 +0100 Subject: [PATCH 048/100] Move SearchHit and SearchHits to Writeable (#37931) This allowed to make SearchHits immutable, while quite a few fields in SearchHit have to stay mutable unfortunately. Relates to #34389 --- .../index/rankeval/RatedSearchHit.java | 3 +- .../org/elasticsearch/search/SearchHit.java | 279 +++++++++--------- .../org/elasticsearch/search/SearchHits.java | 123 ++++---- .../aggregations/metrics/InternalTopHits.java | 2 +- .../search/fetch/FetchSearchResult.java | 4 +- .../internal/InternalSearchResponse.java | 2 +- .../completion/CompletionSuggestion.java | 2 +- .../elasticsearch/search/SearchHitTests.java | 11 +- .../elasticsearch/search/SearchHitsTests.java | 12 +- 9 files changed, 208 insertions(+), 230 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java index 4b76d837b9515..f28e71134bd85 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java @@ -49,8 +49,7 @@ public RatedSearchHit(SearchHit searchHit, OptionalInt rating) { } RatedSearchHit(StreamInput in) throws IOException { - this(SearchHit.readSearchHit(in), - in.readBoolean() == true ? OptionalInt.of(in.readVInt()) : OptionalInt.empty()); + this(new SearchHit(in), in.readBoolean() == true ? OptionalInt.of(in.readVInt()) : OptionalInt.empty()); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/SearchHit.java b/server/src/main/java/org/elasticsearch/search/SearchHit.java index df82bbec59900..4cf3bda83530b 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchHit.java +++ b/server/src/main/java/org/elasticsearch/search/SearchHit.java @@ -32,7 +32,6 @@ import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.ConstructingObjectParser; @@ -80,17 +79,17 @@ * * @see SearchHits */ -public final class SearchHit implements Streamable, ToXContentObject, Iterable { +public final class SearchHit implements Writeable, ToXContentObject, Iterable { - private transient int docId; + private final transient int docId; private static final float DEFAULT_SCORE = Float.NaN; private float score = DEFAULT_SCORE; - private Text id; - private Text type; + private final Text id; + private final Text type; - private NestedIdentity nestedIdentity; + private final NestedIdentity nestedIdentity; private long version = -1; private long seqNo = SequenceNumbers.UNASSIGNED_SEQ_NO; @@ -98,7 +97,7 @@ public final class SearchHit implements Streamable, ToXContentObject, Iterable fields = emptyMap(); + private Map fields; private Map highlightFields = null; @@ -121,10 +120,6 @@ public final class SearchHit implements Streamable, ToXContentObject, Iterable innerHits; - SearchHit() { - - } - //used only in tests public SearchHit(int docId) { this(docId, null, null, null); @@ -146,6 +141,134 @@ public SearchHit(int nestedTopDocId, String id, Text type, NestedIdentity nested this.fields = fields; } + public SearchHit(StreamInput in) throws IOException { + docId = -1; + score = in.readFloat(); + id = in.readOptionalText(); + type = in.readOptionalText(); + nestedIdentity = in.readOptionalWriteable(NestedIdentity::new); + version = in.readLong(); + if (in.getVersion().onOrAfter(Version.V_6_7_0)) { + seqNo = in.readZLong(); + primaryTerm = in.readVLong(); + } + source = in.readBytesReference(); + if (source.length() == 0) { + source = null; + } + if (in.readBoolean()) { + explanation = readExplanation(in); + } + int size = in.readVInt(); + if (size == 0) { + fields = emptyMap(); + } else if (size == 1) { + DocumentField hitField = DocumentField.readDocumentField(in); + fields = singletonMap(hitField.getName(), hitField); + } else { + Map fields = new HashMap<>(); + for (int i = 0; i < size; i++) { + DocumentField hitField = DocumentField.readDocumentField(in); + fields.put(hitField.getName(), hitField); + } + this.fields = unmodifiableMap(fields); + } + + size = in.readVInt(); + if (size == 0) { + highlightFields = emptyMap(); + } else if (size == 1) { + HighlightField field = readHighlightField(in); + highlightFields = singletonMap(field.name(), field); + } else { + Map highlightFields = new HashMap<>(); + for (int i = 0; i < size; i++) { + HighlightField field = readHighlightField(in); + highlightFields.put(field.name(), field); + } + this.highlightFields = unmodifiableMap(highlightFields); + } + + sortValues = new SearchSortValues(in); + + size = in.readVInt(); + if (size > 0) { + matchedQueries = new String[size]; + for (int i = 0; i < size; i++) { + matchedQueries[i] = in.readString(); + } + } + // we call the setter here because that also sets the local index parameter + shard(in.readOptionalWriteable(SearchShardTarget::new)); + size = in.readVInt(); + if (size > 0) { + innerHits = new HashMap<>(size); + for (int i = 0; i < size; i++) { + String key = in.readString(); + SearchHits value = new SearchHits(in); + innerHits.put(key, value); + } + } else { + innerHits = null; + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeFloat(score); + out.writeOptionalText(id); + out.writeOptionalText(type); + out.writeOptionalWriteable(nestedIdentity); + out.writeLong(version); + if (out.getVersion().onOrAfter(Version.V_6_7_0)) { + out.writeZLong(seqNo); + out.writeVLong(primaryTerm); + } + out.writeBytesReference(source); + if (explanation == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + writeExplanation(out, explanation); + } + if (fields == null) { + out.writeVInt(0); + } else { + out.writeVInt(fields.size()); + for (DocumentField hitField : getFields().values()) { + hitField.writeTo(out); + } + } + if (highlightFields == null) { + out.writeVInt(0); + } else { + out.writeVInt(highlightFields.size()); + for (HighlightField highlightField : highlightFields.values()) { + highlightField.writeTo(out); + } + } + sortValues.writeTo(out); + + if (matchedQueries.length == 0) { + out.writeVInt(0); + } else { + out.writeVInt(matchedQueries.length); + for (String matchedFilter : matchedQueries) { + out.writeString(matchedFilter); + } + } + out.writeOptionalWriteable(shard); + if (innerHits == null) { + out.writeVInt(0); + } else { + out.writeVInt(innerHits.size()); + for (Map.Entry entry : innerHits.entrySet()) { + out.writeString(entry.getKey()); + entry.getValue().writeTo(out); + } + } + } + public int docId() { return this.docId; } @@ -771,140 +894,6 @@ private void buildExplanation(XContentBuilder builder, Explanation explanation) builder.endObject(); } - public static SearchHit readSearchHit(StreamInput in) throws IOException { - SearchHit hit = new SearchHit(); - hit.readFrom(in); - return hit; - } - - @Override - public void readFrom(StreamInput in) throws IOException { - score = in.readFloat(); - id = in.readOptionalText(); - type = in.readOptionalText(); - nestedIdentity = in.readOptionalWriteable(NestedIdentity::new); - version = in.readLong(); - if (in.getVersion().onOrAfter(Version.V_6_7_0)) { - seqNo = in.readZLong(); - primaryTerm = in.readVLong(); - } - source = in.readBytesReference(); - if (source.length() == 0) { - source = null; - } - if (in.readBoolean()) { - explanation = readExplanation(in); - } - int size = in.readVInt(); - if (size == 0) { - fields = emptyMap(); - } else if (size == 1) { - DocumentField hitField = DocumentField.readDocumentField(in); - fields = singletonMap(hitField.getName(), hitField); - } else { - Map fields = new HashMap<>(); - for (int i = 0; i < size; i++) { - DocumentField hitField = DocumentField.readDocumentField(in); - fields.put(hitField.getName(), hitField); - } - this.fields = unmodifiableMap(fields); - } - - size = in.readVInt(); - if (size == 0) { - highlightFields = emptyMap(); - } else if (size == 1) { - HighlightField field = readHighlightField(in); - highlightFields = singletonMap(field.name(), field); - } else { - Map highlightFields = new HashMap<>(); - for (int i = 0; i < size; i++) { - HighlightField field = readHighlightField(in); - highlightFields.put(field.name(), field); - } - this.highlightFields = unmodifiableMap(highlightFields); - } - - sortValues = new SearchSortValues(in); - - size = in.readVInt(); - if (size > 0) { - matchedQueries = new String[size]; - for (int i = 0; i < size; i++) { - matchedQueries[i] = in.readString(); - } - } - // we call the setter here because that also sets the local index parameter - shard(in.readOptionalWriteable(SearchShardTarget::new)); - size = in.readVInt(); - if (size > 0) { - innerHits = new HashMap<>(size); - for (int i = 0; i < size; i++) { - String key = in.readString(); - SearchHits value = SearchHits.readSearchHits(in); - innerHits.put(key, value); - } - } else { - innerHits = null; - } - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeFloat(score); - out.writeOptionalText(id); - out.writeOptionalText(type); - out.writeOptionalWriteable(nestedIdentity); - out.writeLong(version); - if (out.getVersion().onOrAfter(Version.V_6_7_0)) { - out.writeZLong(seqNo); - out.writeVLong(primaryTerm); - } - out.writeBytesReference(source); - if (explanation == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - writeExplanation(out, explanation); - } - if (fields == null) { - out.writeVInt(0); - } else { - out.writeVInt(fields.size()); - for (DocumentField hitField : getFields().values()) { - hitField.writeTo(out); - } - } - if (highlightFields == null) { - out.writeVInt(0); - } else { - out.writeVInt(highlightFields.size()); - for (HighlightField highlightField : highlightFields.values()) { - highlightField.writeTo(out); - } - } - sortValues.writeTo(out); - - if (matchedQueries.length == 0) { - out.writeVInt(0); - } else { - out.writeVInt(matchedQueries.length); - for (String matchedFilter : matchedQueries) { - out.writeString(matchedFilter); - } - } - out.writeOptionalWriteable(shard); - if (innerHits == null) { - out.writeVInt(0); - } else { - out.writeVInt(innerHits.size()); - for (Map.Entry entry : innerHits.entrySet()) { - out.writeString(entry.getKey()); - entry.getValue().writeTo(out); - } - } - } - @Override public boolean equals(Object obj) { if (obj == null || getClass() != obj.getClass()) { diff --git a/server/src/main/java/org/elasticsearch/search/SearchHits.java b/server/src/main/java/org/elasticsearch/search/SearchHits.java index f04183ffde700..93478c94048a2 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchHits.java +++ b/server/src/main/java/org/elasticsearch/search/SearchHits.java @@ -26,7 +26,6 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.xcontent.ToXContentFragment; @@ -43,7 +42,7 @@ import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; -public final class SearchHits implements Streamable, ToXContentFragment, Iterable { +public final class SearchHits implements Writeable, ToXContentFragment, Iterable { public static SearchHits empty() { return empty(true); } @@ -55,22 +54,15 @@ public static SearchHits empty(boolean withTotalHits) { public static final SearchHit[] EMPTY = new SearchHit[0]; - private SearchHit[] hits; - - private Total totalHits; - - private float maxScore; - + private final SearchHit[] hits; + private final Total totalHits; + private final float maxScore; @Nullable - private SortField[] sortFields; + private final SortField[] sortFields; @Nullable - private String collapseField; + private final String collapseField; @Nullable - private Object[] collapseValues; - - SearchHits() { - - } + private final Object[] collapseValues; public SearchHits(SearchHit[] hits, @Nullable TotalHits totalHits, float maxScore) { this(hits, totalHits, maxScore, null, null, null); @@ -86,6 +78,55 @@ public SearchHits(SearchHit[] hits, @Nullable TotalHits totalHits, float maxScor this.collapseValues = collapseValues; } + public SearchHits(StreamInput in) throws IOException { + if (in.readBoolean()) { + totalHits = new Total(in); + } else { + // track_total_hits is false + totalHits = null; + } + maxScore = in.readFloat(); + int size = in.readVInt(); + if (size == 0) { + hits = EMPTY; + } else { + hits = new SearchHit[size]; + for (int i = 0; i < hits.length; i++) { + hits[i] = new SearchHit(in); + } + } + if (in.getVersion().onOrAfter(Version.V_6_6_0)) { + sortFields = in.readOptionalArray(Lucene::readSortField, SortField[]::new); + collapseField = in.readOptionalString(); + collapseValues = in.readOptionalArray(Lucene::readSortValue, Object[]::new); + } else { + sortFields = null; + collapseField = null; + collapseValues = null; + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + final boolean hasTotalHits = totalHits != null; + out.writeBoolean(hasTotalHits); + if (hasTotalHits) { + totalHits.writeTo(out); + } + out.writeFloat(maxScore); + out.writeVInt(hits.length); + if (hits.length > 0) { + for (SearchHit hit : hits) { + hit.writeTo(out); + } + } + if (out.getVersion().onOrAfter(Version.V_6_6_0)) { + out.writeOptionalArray(Lucene::writeSortField, sortFields); + out.writeOptionalString(collapseField); + out.writeOptionalArray(Lucene::writeSortValue, collapseValues); + } + } + /** * The total number of hits for the query or null if the tracking of total hits * is disabled in the request. @@ -222,58 +263,6 @@ public static SearchHits fromXContent(XContentParser parser) throws IOException return new SearchHits(hits.toArray(new SearchHit[0]), totalHits, maxScore); } - public static SearchHits readSearchHits(StreamInput in) throws IOException { - SearchHits hits = new SearchHits(); - hits.readFrom(in); - return hits; - } - - @Override - public void readFrom(StreamInput in) throws IOException { - if (in.readBoolean()) { - totalHits = new Total(in); - } else { - // track_total_hits is false - totalHits = null; - } - maxScore = in.readFloat(); - int size = in.readVInt(); - if (size == 0) { - hits = EMPTY; - } else { - hits = new SearchHit[size]; - for (int i = 0; i < hits.length; i++) { - hits[i] = SearchHit.readSearchHit(in); - } - } - if (in.getVersion().onOrAfter(Version.V_6_6_0)) { - sortFields = in.readOptionalArray(Lucene::readSortField, SortField[]::new); - collapseField = in.readOptionalString(); - collapseValues = in.readOptionalArray(Lucene::readSortValue, Object[]::new); - } - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - final boolean hasTotalHits = totalHits != null; - out.writeBoolean(hasTotalHits); - if (hasTotalHits) { - totalHits.writeTo(out); - } - out.writeFloat(maxScore); - out.writeVInt(hits.length); - if (hits.length > 0) { - for (SearchHit hit : hits) { - hit.writeTo(out); - } - } - if (out.getVersion().onOrAfter(Version.V_6_6_0)) { - out.writeOptionalArray(Lucene::writeSortField, sortFields); - out.writeOptionalString(collapseField); - out.writeOptionalArray(Lucene::writeSortValue, collapseValues); - } - } - @Override public boolean equals(Object obj) { if (obj == null || getClass() != obj.getClass()) { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTopHits.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTopHits.java index 60ae4c78edc7e..4a266ee0703bd 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTopHits.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTopHits.java @@ -65,7 +65,7 @@ public InternalTopHits(StreamInput in) throws IOException { from = in.readVInt(); size = in.readVInt(); topDocs = Lucene.readTopDocs(in); - searchHits = SearchHits.readSearchHits(in); + searchHits = new SearchHits(in); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchSearchResult.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchSearchResult.java index 12391151861d0..400ab3623c0e8 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchSearchResult.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchSearchResult.java @@ -22,9 +22,9 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchShardTarget; -import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.query.QuerySearchResult; import java.io.IOException; @@ -92,7 +92,7 @@ public static FetchSearchResult readFetchSearchResult(StreamInput in) throws IOE public void readFrom(StreamInput in) throws IOException { super.readFrom(in); requestId = in.readLong(); - hits = SearchHits.readSearchHits(in); + hits = new SearchHits(in); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java b/server/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java index e78ce7f3fb194..bac7b6486bb88 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java +++ b/server/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java @@ -51,7 +51,7 @@ public InternalSearchResponse(SearchHits hits, InternalAggregations aggregations public InternalSearchResponse(StreamInput in) throws IOException { super( - SearchHits.readSearchHits(in), + new SearchHits(in), in.readBoolean() ? InternalAggregations.readAggregations(in) : null, in.readBoolean() ? new Suggest(in) : null, in.readBoolean(), diff --git a/server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestion.java b/server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestion.java index 34c191df3b36a..bc2e2abdb7d03 100644 --- a/server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestion.java +++ b/server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestion.java @@ -292,7 +292,7 @@ public Option(StreamInput in) throws IOException { super(in); this.doc = Lucene.readScoreDoc(in); if (in.readBoolean()) { - this.hit = SearchHit.readSearchHit(in); + this.hit = new SearchHit(in); } int contextSize = in.readInt(); this.contexts = new LinkedHashMap<>(contextSize); diff --git a/server/src/test/java/org/elasticsearch/search/SearchHitTests.java b/server/src/test/java/org/elasticsearch/search/SearchHitTests.java index 4831729201183..bbd5b12ec45ee 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchHitTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchHitTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.document.DocumentField; +import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -39,7 +40,7 @@ import org.elasticsearch.search.SearchHit.NestedIdentity; import org.elasticsearch.search.fetch.subphase.highlight.HighlightField; import org.elasticsearch.search.fetch.subphase.highlight.HighlightFieldTests; -import org.elasticsearch.test.AbstractStreamableTestCase; +import org.elasticsearch.test.AbstractWireSerializingTestCase; import org.elasticsearch.test.RandomObjects; import org.elasticsearch.test.VersionUtils; @@ -59,7 +60,7 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; -public class SearchHitTests extends AbstractStreamableTestCase { +public class SearchHitTests extends AbstractWireSerializingTestCase { public static SearchHit createTestItem(boolean withOptionalInnerHits, boolean withShardTarget) { return createTestItem(randomFrom(XContentType.values()), withOptionalInnerHits, withShardTarget); } @@ -139,8 +140,8 @@ public static SearchHit createTestItem(XContentType xContentType, boolean withOp } @Override - protected SearchHit createBlankInstance() { - return new SearchHit(); + protected Writeable.Reader instanceReader() { + return SearchHit::new; } @Override @@ -246,7 +247,7 @@ public void testSerializeShardTarget() throws Exception { SearchHits hits = new SearchHits(new SearchHit[]{hit1, hit2}, new TotalHits(2, TotalHits.Relation.EQUAL_TO), 1f); Version version = VersionUtils.randomVersion(random()); - SearchHits results = copyStreamable(hits, getNamedWriteableRegistry(), SearchHits::new, version); + SearchHits results = copyWriteable(hits, getNamedWriteableRegistry(), SearchHits::new, version); SearchShardTarget deserializedTarget = results.getAt(0).getShard(); assertThat(deserializedTarget, equalTo(target)); assertThat(results.getAt(0).getInnerHits().get("1").getAt(0).getShard(), notNullValue()); diff --git a/server/src/test/java/org/elasticsearch/search/SearchHitsTests.java b/server/src/test/java/org/elasticsearch/search/SearchHitsTests.java index 396879e8f65bd..9e87628d35d1c 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchHitsTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchHitsTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.lucene.LuceneTests; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; @@ -37,7 +38,7 @@ import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.test.AbstractStreamableXContentTestCase; +import org.elasticsearch.test.AbstractSerializingTestCase; import org.elasticsearch.test.VersionUtils; import java.io.IOException; @@ -45,7 +46,7 @@ import java.util.Collections; import java.util.function.Predicate; -public class SearchHitsTests extends AbstractStreamableXContentTestCase { +public class SearchHitsTests extends AbstractSerializingTestCase { public static SearchHits createTestItem(boolean withOptionalInnerHits, boolean withShardTarget) { return createTestItem(randomFrom(XContentType.values()), withOptionalInnerHits, withShardTarget); @@ -171,8 +172,8 @@ protected String[] getShuffleFieldsExceptions() { } @Override - protected SearchHits createBlankInstance() { - return new SearchHits(); + protected Writeable.Reader instanceReader() { + return SearchHits::new; } @Override @@ -274,8 +275,7 @@ public void testFromXContentWithShards() throws IOException { public void testReadFromPre6_6_0() throws IOException { try (StreamInput in = StreamInput.wrap(Base64.getDecoder().decode("AQC/gAAAAAA="))) { in.setVersion(VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, VersionUtils.getPreviousVersion(Version.V_6_6_0))); - SearchHits searchHits = new SearchHits(); - searchHits.readFrom(in); + SearchHits searchHits = new SearchHits(in); assertEquals(0, searchHits.getHits().length); assertNotNull(searchHits.getTotalHits()); assertEquals(0L, searchHits.getTotalHits().value); From f3379940c6ceb74a3bcb553c90151e1650a2718d Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Wed, 30 Jan 2019 14:41:35 +0200 Subject: [PATCH 049/100] Fix the packer cache script (#38023) The script is used to create a cache on ephemeral CI workers. Changes: - create and use a `pullFixture` task that always exists regardless of docker support - wire dependencies correctly so any pre fixture setup runs for pull as well - set up java env vars so bwc versions can build --- .ci/packer_cache.sh | 7 ++++++- .../gradle/testfixtures/TestFixturesPlugin.java | 5 +++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.ci/packer_cache.sh b/.ci/packer_cache.sh index 6a7e2e69faebf..ce78ebcc1cae7 100755 --- a/.ci/packer_cache.sh +++ b/.ci/packer_cache.sh @@ -16,4 +16,9 @@ while [ -h "$SCRIPT" ] ; do done source $(dirname "${SCRIPT}")/java-versions.properties -JAVA_HOME="${HOME}"/.java/${ES_BUILD_JAVA} ./gradlew --parallel resolveAllDependencies composePull +export JAVA_HOME="${HOME}"/.java/${ES_BUILD_JAVA} +# We are caching BWC versions too, need these so we can build those +export JAVA8_HOME="${HOME}"/.java/java8 +export JAVA11_HOME="${HOME}"/.java/java11 +export JAVA12_HOME="${HOME}"/.java/java12 +./gradlew --parallel clean pullFixture --scan -Porg.elasticsearch.acceptScanTOS=true -s resolveAllDependencies diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java index 35a7eacf1fde1..3dfccaf435031 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java @@ -59,14 +59,17 @@ public void apply(Project project) { disableTaskByType(tasks, JarHellTask.class); Task buildFixture = project.getTasks().create("buildFixture"); + Task pullFixture = project.getTasks().create("pullFixture"); Task preProcessFixture = project.getTasks().create("preProcessFixture"); buildFixture.dependsOn(preProcessFixture); + pullFixture.dependsOn(preProcessFixture); Task postProcessFixture = project.getTasks().create("postProcessFixture"); if (dockerComposeSupported(project) == false) { preProcessFixture.setEnabled(false); postProcessFixture.setEnabled(false); buildFixture.setEnabled(false); + pullFixture.setEnabled(false); return; } @@ -81,7 +84,9 @@ public void apply(Project project) { ); buildFixture.dependsOn(tasks.getByName("composeUp")); + pullFixture.dependsOn(tasks.getByName("composePull")); tasks.getByName("composeUp").mustRunAfter(preProcessFixture); + tasks.getByName("composePull").mustRunAfter(preProcessFixture); postProcessFixture.dependsOn(buildFixture); configureServiceInfoForTask( From 8280a2066444391979a7a35a73c39a0d19c3186e Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Wed, 30 Jan 2019 06:51:11 -0600 Subject: [PATCH 050/100] ML: Add upgrade mode docs, hlrc, and fix bug (#37942) * ML: Add upgrade mode docs, hlrc, and fix bug * [DOCS] Fixes build error and edits text * adjusting docs * Update docs/reference/ml/apis/set-upgrade-mode.asciidoc Co-Authored-By: benwtrent * Update set-upgrade-mode.asciidoc * Update set-upgrade-mode.asciidoc --- .../client/MLRequestConverters.java | 12 ++ .../client/MachineLearningClient.java | 39 +++++++ .../client/ml/SetUpgradeModeRequest.java | 93 ++++++++++++++++ .../client/MLRequestConvertersTests.java | 17 +++ .../client/MachineLearningIT.java | 27 +++++ .../MlClientDocumentationIT.java | 52 +++++++++ .../high-level/ml/set-upgrade-mode.asciidoc | 40 +++++++ .../high-level/supported-apis.asciidoc | 2 + docs/reference/ml/apis/ml-api.asciidoc | 13 ++- .../ml/apis/set-upgrade-mode.asciidoc | 103 ++++++++++++++++++ .../xpack/core/ml/MlMetadata.java | 8 +- .../action/TransportSetUpgradeModeAction.java | 7 ++ .../api/ml.set_upgrade_mode.json | 2 +- 13 files changed, 407 insertions(+), 8 deletions(-) create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/SetUpgradeModeRequest.java create mode 100644 docs/java-rest/high-level/ml/set-upgrade-mode.asciidoc create mode 100644 docs/reference/ml/apis/set-upgrade-mode.asciidoc diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java index 6b0a5d2642f02..073b92f84a3a3 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java @@ -64,6 +64,7 @@ import org.elasticsearch.client.ml.PutFilterRequest; import org.elasticsearch.client.ml.PutJobRequest; import org.elasticsearch.client.ml.RevertModelSnapshotRequest; +import org.elasticsearch.client.ml.SetUpgradeModeRequest; import org.elasticsearch.client.ml.StartDatafeedRequest; import org.elasticsearch.client.ml.StopDatafeedRequest; import org.elasticsearch.client.ml.UpdateDatafeedRequest; @@ -624,6 +625,17 @@ static Request deleteFilter(DeleteFilterRequest deleteFilterRequest) { return request; } + static Request setUpgradeMode(SetUpgradeModeRequest setUpgradeModeRequest) { + String endpoint = new EndpointBuilder().addPathPartAsIs("_ml", "set_upgrade_mode").build(); + Request request = new Request(HttpPost.METHOD_NAME, endpoint); + RequestConverters.Params params = new RequestConverters.Params(request); + params.putParam(SetUpgradeModeRequest.ENABLED.getPreferredName(), Boolean.toString(setUpgradeModeRequest.isEnabled())); + if (setUpgradeModeRequest.getTimeout() != null) { + params.putParam(SetUpgradeModeRequest.TIMEOUT.getPreferredName(), setUpgradeModeRequest.getTimeout().toString()); + } + return request; + } + static Request mlInfo(MlInfoRequest infoRequest) { String endpoint = new EndpointBuilder() .addPathPartAsIs("_ml", "info") diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java index aaff35a238998..2e359931c1025 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java @@ -86,6 +86,7 @@ import org.elasticsearch.client.ml.PutJobResponse; import org.elasticsearch.client.ml.RevertModelSnapshotRequest; import org.elasticsearch.client.ml.RevertModelSnapshotResponse; +import org.elasticsearch.client.ml.SetUpgradeModeRequest; import org.elasticsearch.client.ml.StartDatafeedRequest; import org.elasticsearch.client.ml.StartDatafeedResponse; import org.elasticsearch.client.ml.StopDatafeedRequest; @@ -1838,4 +1839,42 @@ public void findFileStructureAsync(FindFileStructureRequest request, RequestOpti listener, Collections.emptySet()); } + + /** + * Sets the ML cluster setting upgrade_mode + *

+ * For additional info + * see Set Upgrade Mode + * + * @param request The request to set upgrade mode + * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return response + * @throws IOException when there is a serialization issue sending the request or receiving the response + */ + public AcknowledgedResponse setUpgradeMode(SetUpgradeModeRequest request, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(request, + MLRequestConverters::setUpgradeMode, + options, + AcknowledgedResponse::fromXContent, + Collections.emptySet()); + } + + /** + * Sets the ML cluster setting upgrade_mode asynchronously + *

+ * For additional info + * see Set Upgrade Mode + * + * @param request The request of Machine Learning info + * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener Listener to be notified upon request completion + */ + public void setUpgradeModeAsync(SetUpgradeModeRequest request, RequestOptions options, ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(request, + MLRequestConverters::setUpgradeMode, + options, + AcknowledgedResponse::fromXContent, + listener, + Collections.emptySet()); + } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/SetUpgradeModeRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/SetUpgradeModeRequest.java new file mode 100644 index 0000000000000..64e94f0251785 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/SetUpgradeModeRequest.java @@ -0,0 +1,93 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.ml; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.unit.TimeValue; + +import java.util.Objects; + +/** + * Sets ML into upgrade_mode + */ +public class SetUpgradeModeRequest extends ActionRequest { + + + public static final ParseField ENABLED = new ParseField("enabled"); + public static final ParseField TIMEOUT = new ParseField("timeout"); + + private boolean enabled; + private TimeValue timeout; + + /** + * Create a new request + * + * @param enabled whether to enable `upgrade_mode` or not + */ + public SetUpgradeModeRequest(boolean enabled) { + this.enabled = enabled; + } + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public TimeValue getTimeout() { + return timeout; + } + + /** + * How long to wait for the request to be completed + * + * @param timeout default value of 30 seconds + */ + public void setTimeout(TimeValue timeout) { + this.timeout = timeout; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public int hashCode() { + return Objects.hash(enabled, timeout); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other == null || getClass() != other.getClass()) { + return false; + } + + SetUpgradeModeRequest that = (SetUpgradeModeRequest) other; + return Objects.equals(enabled, that.enabled) && Objects.equals(timeout, that.timeout); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java index b472837f9c25d..11faaf879729d 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java @@ -61,6 +61,7 @@ import org.elasticsearch.client.ml.PutFilterRequest; import org.elasticsearch.client.ml.PutJobRequest; import org.elasticsearch.client.ml.RevertModelSnapshotRequest; +import org.elasticsearch.client.ml.SetUpgradeModeRequest; import org.elasticsearch.client.ml.StartDatafeedRequest; import org.elasticsearch.client.ml.StartDatafeedRequestTests; import org.elasticsearch.client.ml.StopDatafeedRequest; @@ -818,6 +819,22 @@ public void testFindFileStructure() throws Exception { assertEquals(sample, requestEntityToString(request)); } + public void testSetUpgradeMode() { + SetUpgradeModeRequest setUpgradeModeRequest = new SetUpgradeModeRequest(true); + + Request request = MLRequestConverters.setUpgradeMode(setUpgradeModeRequest); + assertThat(request.getEndpoint(), equalTo("/_ml/set_upgrade_mode")); + assertThat(request.getMethod(), equalTo(HttpPost.METHOD_NAME)); + assertThat(request.getParameters().get(SetUpgradeModeRequest.ENABLED.getPreferredName()), equalTo(Boolean.toString(true))); + assertThat(request.getParameters().containsKey(SetUpgradeModeRequest.TIMEOUT.getPreferredName()), is(false)); + + setUpgradeModeRequest.setTimeout(TimeValue.timeValueHours(1)); + setUpgradeModeRequest.setEnabled(false); + request = MLRequestConverters.setUpgradeMode(setUpgradeModeRequest); + assertThat(request.getParameters().get(SetUpgradeModeRequest.ENABLED.getPreferredName()), equalTo(Boolean.toString(false))); + assertThat(request.getParameters().get(SetUpgradeModeRequest.TIMEOUT.getPreferredName()), is("1h")); + } + private static Job createValidJob(String jobId) { AnalysisConfig.Builder analysisConfig = AnalysisConfig.builder(Collections.singletonList( Detector.builder().setFunction("count").build())); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java index 162fceed9e572..07d7187fd1d70 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java @@ -84,6 +84,7 @@ import org.elasticsearch.client.ml.PutJobResponse; import org.elasticsearch.client.ml.RevertModelSnapshotRequest; import org.elasticsearch.client.ml.RevertModelSnapshotResponse; +import org.elasticsearch.client.ml.SetUpgradeModeRequest; import org.elasticsearch.client.ml.StartDatafeedRequest; import org.elasticsearch.client.ml.StartDatafeedResponse; import org.elasticsearch.client.ml.StopDatafeedRequest; @@ -1614,4 +1615,30 @@ public void testFindFileStructure() throws IOException { assertEquals("timestamp", structure.getTimestampField()); assertFalse(structure.needClientTimezone()); } + + public void testEnableUpgradeMode() throws Exception { + MachineLearningClient machineLearningClient = highLevelClient().machineLearning(); + + MlInfoResponse mlInfoResponse = machineLearningClient.getMlInfo(new MlInfoRequest(), RequestOptions.DEFAULT); + assertThat(mlInfoResponse.getInfo().get("upgrade_mode"), equalTo(false)); + + AcknowledgedResponse setUpgrademodeResponse = execute(new SetUpgradeModeRequest(true), + machineLearningClient::setUpgradeMode, + machineLearningClient::setUpgradeModeAsync); + + assertThat(setUpgrademodeResponse.isAcknowledged(), is(true)); + + + mlInfoResponse = machineLearningClient.getMlInfo(new MlInfoRequest(), RequestOptions.DEFAULT); + assertThat(mlInfoResponse.getInfo().get("upgrade_mode"), equalTo(true)); + + setUpgrademodeResponse = execute(new SetUpgradeModeRequest(false), + machineLearningClient::setUpgradeMode, + machineLearningClient::setUpgradeModeAsync); + + assertThat(setUpgrademodeResponse.isAcknowledged(), is(true)); + + mlInfoResponse = machineLearningClient.getMlInfo(new MlInfoRequest(), RequestOptions.DEFAULT); + assertThat(mlInfoResponse.getInfo().get("upgrade_mode"), equalTo(false)); + } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java index 68881206b487f..f4e3f86196f29 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java @@ -99,6 +99,7 @@ import org.elasticsearch.client.ml.PutJobResponse; import org.elasticsearch.client.ml.RevertModelSnapshotRequest; import org.elasticsearch.client.ml.RevertModelSnapshotResponse; +import org.elasticsearch.client.ml.SetUpgradeModeRequest; import org.elasticsearch.client.ml.StartDatafeedRequest; import org.elasticsearch.client.ml.StartDatafeedResponse; import org.elasticsearch.client.ml.StopDatafeedRequest; @@ -3078,6 +3079,57 @@ public void onFailure(Exception e) { } } + public void testSetUpgradeMode() throws Exception { + RestHighLevelClient client = highLevelClient(); + { + // tag::set-upgrade-mode-request + SetUpgradeModeRequest request = new SetUpgradeModeRequest(true); // <1> + request.setTimeout(TimeValue.timeValueMinutes(10)); // <2> + // end::set-upgrade-mode-request + + // Set to false so that the cluster setting does not have to be unset at the end of the test. + request.setEnabled(false); + + // tag::set-upgrade-mode-execute + AcknowledgedResponse acknowledgedResponse = client.machineLearning().setUpgradeMode(request, RequestOptions.DEFAULT); + // end::set-upgrade-mode-execute + + // tag::set-upgrade-mode-response + boolean acknowledged = acknowledgedResponse.isAcknowledged(); // <1> + // end::set-upgrade-mode-response + assertThat(acknowledged, is(true)); + } + { + SetUpgradeModeRequest request = new SetUpgradeModeRequest(false); + + // tag::set-upgrade-mode-execute-listener + ActionListener listener = new ActionListener() { + @Override + public void onResponse(AcknowledgedResponse acknowledgedResponse) { + // <1> + } + + @Override + public void onFailure(Exception e) { + // <2> + } + }; + // end::set-upgrade-mode-execute-listener + + // Replace the empty listener by a blocking listener in test + final CountDownLatch latch = new CountDownLatch(1); + listener = new LatchedActionListener<>(listener, latch); + + // tag::set-upgrade-mode-execute-async + client.machineLearning() + .setUpgradeModeAsync(request, RequestOptions.DEFAULT, listener); // <1> + // end::set-upgrade-mode-execute-async + + assertTrue(latch.await(30L, TimeUnit.SECONDS)); + + } + } + private String createFilter(RestHighLevelClient client) throws IOException { MlFilter.Builder filterBuilder = MlFilter.builder("my_safe_domains") .setDescription("A list of safe domains") diff --git a/docs/java-rest/high-level/ml/set-upgrade-mode.asciidoc b/docs/java-rest/high-level/ml/set-upgrade-mode.asciidoc new file mode 100644 index 0000000000000..80bb1874e4a63 --- /dev/null +++ b/docs/java-rest/high-level/ml/set-upgrade-mode.asciidoc @@ -0,0 +1,40 @@ +-- +:api: set-upgrade-mode +:request: SetUpgradeModeRequest +:response: AcknowledgedResponse +-- +[id="{upid}-{api}"] +=== Set Upgrade Mode API + +The Set Upgrade Mode API temporarily halts all {ml} job and {dfeed} tasks when `enabled=true`. Their +reported states remain unchanged. Consequently, when exiting upgrade mode the halted {ml} jobs and +{dfeeds} will return to their previous state. + +It accepts a +{request}+ object and responds with a +{response}+ object. + +When `enabled=true`, no new jobs can be opened, and no job or {dfeed} tasks will +be running. Be sure to set `enabled=false` once upgrade actions are completed. + +[id="{upid}-{api}-request"] +==== Set Upgrade Mode Request + +A +{request}+ object gets created setting the desired `enabled` state. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-request] +-------------------------------------------------- +<1> Constructing a new request referencing enabling upgrade mode +<2> Optionally setting the `timeout` value for how long the +execution should wait. + +[id="{upid}-{api}-response"] +==== Set Upgrade Mode Response + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-response] +-------------------------------------------------- +<1> `isAcknowledged()` from the +{response}+ indicates if the setting was set successfully. + +include::../execution.asciidoc[] \ No newline at end of file diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc index 0b4a2570c896d..70f06e457e9b5 100644 --- a/docs/java-rest/high-level/supported-apis.asciidoc +++ b/docs/java-rest/high-level/supported-apis.asciidoc @@ -295,6 +295,7 @@ The Java High Level REST Client supports the following Machine Learning APIs: * <<{upid}-update-model-snapshot>> * <<{upid}-get-ml-info>> * <<{upid}-delete-expired-data>> +* <<{upid}-set-upgrade-mode>> include::ml/put-job.asciidoc[] include::ml/get-job.asciidoc[] @@ -338,6 +339,7 @@ include::ml/revert-model-snapshot.asciidoc[] include::ml/update-model-snapshot.asciidoc[] include::ml/get-info.asciidoc[] include::ml/delete-expired-data.asciidoc[] +include::ml/set-upgrade-mode.asciidoc[] == Migration APIs diff --git a/docs/reference/ml/apis/ml-api.asciidoc b/docs/reference/ml/apis/ml-api.asciidoc index 6cb0dc6ba4093..7933dea85ce0a 100644 --- a/docs/reference/ml/apis/ml-api.asciidoc +++ b/docs/reference/ml/apis/ml-api.asciidoc @@ -72,7 +72,7 @@ machine learning APIs and in advanced job configuration options in Kibana. [float] [[ml-api-file-structure-endpoint]] -=== File Structure +=== File structure * <> @@ -84,10 +84,16 @@ machine learning APIs and in advanced job configuration options in Kibana. [float] [[ml-api-delete-expired-data-endpoint]] -=== Delete Expired Data +=== Delete expired data * <> +[float] +[[ml-set-upgrade-mode-endpoint]] +=== Set upgrade mode + +* <> + //ADD include::post-calendar-event.asciidoc[] include::put-calendar-job.asciidoc[] @@ -137,7 +143,8 @@ include::post-data.asciidoc[] include::preview-datafeed.asciidoc[] //REVERT include::revert-snapshot.asciidoc[] -//START/STOP +//SET/START/STOP +include::set-upgrade-mode.asciidoc[] include::start-datafeed.asciidoc[] include::stop-datafeed.asciidoc[] //UPDATE diff --git a/docs/reference/ml/apis/set-upgrade-mode.asciidoc b/docs/reference/ml/apis/set-upgrade-mode.asciidoc new file mode 100644 index 0000000000000..5434d70d4e61e --- /dev/null +++ b/docs/reference/ml/apis/set-upgrade-mode.asciidoc @@ -0,0 +1,103 @@ +[role="xpack"] +[testenv="platinum"] +[[ml-set-upgrade-mode]] +=== Set upgrade mode API +++++ +Set upgrade mode +++++ + +Sets a cluster wide upgrade_mode setting that prepares {ml} indices for an +upgrade. + +==== Request +////////////////////////// + +[source,js] +-------------------------------------------------- +POST /_ml/set_upgrade_mode?enabled=false&timeout=10m +-------------------------------------------------- +// CONSOLE +// TEST +// TEARDOWN + +////////////////////////// + + +`POST _ml/set_upgrade_mode` + +==== Description + +When upgrading your cluster, in some circumstances you must restart your nodes and +reindex your {ml} indices. In those circumstances, there must be no {ml} jobs running. +You can close the {ml} jobs, do the upgrade, then open all the jobs again. +Alternatively, you can use this API to temporarily halt tasks associated +with the jobs and {dfeeds} and prevent new jobs from opening. You can also use this +API during upgrades that do not require you to reindex your {ml} indices, +though stopping jobs is not a requirement in that case. + +For more information, see {stack-ref}/upgrading-elastic-stack.html[Upgrading the {stack}]. + + +When `enabled=true` this API temporarily halts all job and {dfeed} tasks and +prohibits new job and {dfeed} tasks from starting. + +Subsequently, you can call the API with the enabled parameter set to false, +which causes {ml} jobs and {dfeeds} to return to their desired states. + +You can see the current value for the `upgrade_mode` setting by using the +<>. + +IMPORTANT: No new {ml} jobs can be opened while the `upgrade_mode` setting is +`true`. + +==== Query Parameters + +`enabled`:: + (boolean) When `true`, this enables `upgrade_mode`. Defaults to `false` + +`timeout`:: + (time) The time to wait for the request to be completed. + The default value is 30 seconds. + +==== Authorization + +You must have `manage_ml`, or `manage` cluster privileges to use this API. +For more information, see +{stack-ov}/security-privileges.html[Security privileges]. + + +==== Examples + +The following example enables `upgrade_mode` for the cluster: + +[source,js] +-------------------------------------------------- +POST _ml/set_upgrade_mode?enabled=true&timeout=10m +-------------------------------------------------- +// CONSOLE +// TEST + +When the call is successful, an acknowledged response is returned. For example: + +[source,js] +---- +{ + "acknowledged": true +} +---- +// TESTRESPONSE + +The acknowledged response will only be returned once all {ml} jobs and {dfeeds} have +finished writing to the {ml} internal indices. This means it is safe to reindex those +internal indices without causing failures. You must wait for the acknowledged +response before reindexing to ensure that all writes are completed. + +When the upgrade is complete, you must set `upgrade_mode` to `false` for +{ml} jobs to start running again. For example: + +[source,js] +-------------------------------------------------- +POST _ml/set_upgrade_mode?enabled=false&timeout=10m +-------------------------------------------------- +// CONSOLE +// TEST diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java index 43462c552da63..54c83e9a88a75 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java @@ -137,7 +137,7 @@ public MlMetadata(StreamInput in) throws IOException { } this.datafeeds = datafeeds; this.groupOrJobLookup = new GroupOrJobLookup(jobs.values()); - if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + if (in.getVersion().onOrAfter(Version.V_6_7_0)) { this.upgradeMode = in.readBoolean(); } else { this.upgradeMode = false; @@ -148,7 +148,7 @@ public MlMetadata(StreamInput in) throws IOException { public void writeTo(StreamOutput out) throws IOException { writeMap(jobs, out); writeMap(datafeeds, out); - if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + if (out.getVersion().onOrAfter(Version.V_6_7_0)) { out.writeBoolean(upgradeMode); } } @@ -201,7 +201,7 @@ public MlMetadataDiff(StreamInput in) throws IOException { MlMetadataDiff::readJobDiffFrom); this.datafeeds = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), DatafeedConfig::new, MlMetadataDiff::readDatafeedDiffFrom); - if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + if (in.getVersion().onOrAfter(Version.V_6_7_0)) { upgradeMode = in.readBoolean(); } else { upgradeMode = false; @@ -224,7 +224,7 @@ public MetaData.Custom apply(MetaData.Custom part) { public void writeTo(StreamOutput out) throws IOException { jobs.writeTo(out); datafeeds.writeTo(out); - if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + if (out.getVersion().onOrAfter(Version.V_6_7_0)) { out.writeBoolean(upgradeMode); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSetUpgradeModeAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSetUpgradeModeAction.java index 1834b0b3c0616..6ec77a34c55b6 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSetUpgradeModeAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSetUpgradeModeAction.java @@ -164,6 +164,13 @@ protected void masterOperation(SetUpgradeModeAction.Request request, ClusterStat wrappedListener.onFailure(new ElasticsearchTimeoutException("Unknown error occurred while updating cluster state")); return; } + + // There are no tasks to worry about starting/stopping + if (tasksCustomMetaData == null || tasksCustomMetaData.tasks().isEmpty()) { + wrappedListener.onResponse(new AcknowledgedResponse(true)); + return; + } + // Did we change from disabled -> enabled? if (request.isEnabled()) { isolateDatafeeds(tasksCustomMetaData, isolateDatafeedListener); diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.set_upgrade_mode.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.set_upgrade_mode.json index bb3220ece6b13..5406d2d06eceb 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.set_upgrade_mode.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.set_upgrade_mode.json @@ -1,6 +1,6 @@ { "ml.set_upgrade_mode": { - "documentation": "TODO", + "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-set-upgrade-mode.html", "methods": [ "POST" ], "url": { "path": "/_ml/set_upgrade_mode", From 53e80e98148a714d90da8f49bb283d290d737109 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Wed, 30 Jan 2019 16:11:44 +0200 Subject: [PATCH 051/100] Fix failure in test code ClusterPrivilegeTests Closes #38030 --- .../org/elasticsearch/integration/ClusterPrivilegeTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClusterPrivilegeTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClusterPrivilegeTests.java index 3b30982784b76..820dba9b96b6d 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClusterPrivilegeTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClusterPrivilegeTests.java @@ -168,6 +168,7 @@ public void testThatSnapshotAndRestore() throws Exception { waitForSnapshotToFinish("my-repo", "my-snapshot"); // user_d can create snapshots, but not concurrently assertAccessIsAllowed("user_d", "PUT", "/_snapshot/my-repo/my-snapshot-d", "{ \"indices\": \"someindex\" }"); + waitForSnapshotToFinish("my-repo", "my-snapshot-d"); assertAccessIsDenied("user_a", "DELETE", "/someindex"); assertAccessIsDenied("user_b", "DELETE", "/someindex"); From 23805fa41ad40e02611a765f96b2188a1e730e64 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 30 Jan 2019 09:20:30 -0500 Subject: [PATCH 052/100] Geo: Fix Empty Geometry Collection Handling (#37978) Fixes handling empty geometry collection and re-enables testParseGeometryCollection test. Fixes #37894 --- .../geo/utils/WellKnownText.java | 4 ++ .../builders/GeometryCollectionBuilder.java | 4 ++ .../geo/builders/MultiLineStringBuilder.java | 3 + .../geo/builders/MultiPointBuilder.java | 10 ++++ .../geo/builders/MultiPolygonBuilder.java | 3 + .../common/geo/parsers/GeoWKTParser.java | 4 +- .../common/geo/GeoWKTShapeParserTests.java | 57 +++++++++++++------ 7 files changed, 65 insertions(+), 20 deletions(-) diff --git a/libs/geo/src/main/java/org/elasticsearch/geo/utils/WellKnownText.java b/libs/geo/src/main/java/org/elasticsearch/geo/utils/WellKnownText.java index 5fa585be28b24..c6e96d9bdf3ce 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geo/utils/WellKnownText.java +++ b/libs/geo/src/main/java/org/elasticsearch/geo/utils/WellKnownText.java @@ -121,6 +121,10 @@ public Void visit(MultiLine multiLine) { @Override public Void visit(MultiPoint multiPoint) { + if (multiPoint.isEmpty()) { + sb.append(EMPTY); + return null; + } // walk through coordinates: sb.append(LPAREN); visitPoint(multiPoint.get(0).getLon(), multiPoint.get(0).getLat()); diff --git a/server/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java b/server/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java index fb3ff6203ed45..fb0cacacfae84 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java +++ b/server/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.geo.geometry.GeometryCollection; import org.locationtech.spatial4j.shape.Shape; import java.io.IOException; @@ -186,6 +187,9 @@ public Shape buildS4J() { @Override public org.elasticsearch.geo.geometry.GeometryCollection buildGeometry() { + if (this.shapes.isEmpty()) { + return GeometryCollection.EMPTY; + } List shapes = new ArrayList<>(this.shapes.size()); for (ShapeBuilder shape : this.shapes) { diff --git a/server/src/main/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilder.java b/server/src/main/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilder.java index a283cda874528..24a8b3b226f36 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilder.java +++ b/server/src/main/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilder.java @@ -151,6 +151,9 @@ public JtsGeometry buildS4J() { @Override public org.elasticsearch.geo.geometry.Geometry buildGeometry() { + if (lines.isEmpty()) { + return MultiLine.EMPTY; + } if (wrapdateline) { List parts = new ArrayList<>(); for (LineStringBuilder line : lines) { diff --git a/server/src/main/java/org/elasticsearch/common/geo/builders/MultiPointBuilder.java b/server/src/main/java/org/elasticsearch/common/geo/builders/MultiPointBuilder.java index c92d67e8291ea..360447a963e80 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/builders/MultiPointBuilder.java +++ b/server/src/main/java/org/elasticsearch/common/geo/builders/MultiPointBuilder.java @@ -45,6 +45,13 @@ public MultiPointBuilder(List coordinates) { super(coordinates); } + /** + * Creates a new empty MultiPoint builder + */ + public MultiPointBuilder() { + super(); + } + /** * Read from a stream. */ @@ -77,6 +84,9 @@ public XShapeCollection buildS4J() { @Override public MultiPoint buildGeometry() { + if (coordinates.isEmpty()) { + return MultiPoint.EMPTY; + } return new MultiPoint(coordinates.stream().map(coord -> new org.elasticsearch.geo.geometry.Point(coord.y, coord.x)) .collect(Collectors.toList())); } diff --git a/server/src/main/java/org/elasticsearch/common/geo/builders/MultiPolygonBuilder.java b/server/src/main/java/org/elasticsearch/common/geo/builders/MultiPolygonBuilder.java index be0741306c097..466f96c78ec8d 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/builders/MultiPolygonBuilder.java +++ b/server/src/main/java/org/elasticsearch/common/geo/builders/MultiPolygonBuilder.java @@ -198,6 +198,9 @@ public MultiPolygon buildGeometry() { shapes.add((org.elasticsearch.geo.geometry.Polygon)poly); } } + if (shapes.isEmpty()) { + return MultiPolygon.EMPTY; + } return new MultiPolygon(shapes); } diff --git a/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoWKTParser.java b/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoWKTParser.java index bf26980c92651..2cffa417246fd 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoWKTParser.java +++ b/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoWKTParser.java @@ -198,7 +198,7 @@ private static MultiPointBuilder parseMultiPoint(StreamTokenizer stream, final b throws IOException, ElasticsearchParseException { String token = nextEmptyOrOpen(stream); if (token.equals(EMPTY)) { - return null; + return new MultiPointBuilder(); } return new MultiPointBuilder(parseCoordinateList(stream, ignoreZValue, coerce)); } @@ -242,7 +242,7 @@ private static MultiLineStringBuilder parseMultiLine(StreamTokenizer stream, fin throws IOException, ElasticsearchParseException { String token = nextEmptyOrOpen(stream); if (token.equals(EMPTY)) { - return null; + return new MultiLineStringBuilder(); } MultiLineStringBuilder builder = new MultiLineStringBuilder(); builder.linestring(parseLine(stream, ignoreZValue, coerce)); diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java index 6518e05cf330c..286e1ce6ee7c5 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java @@ -40,6 +40,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.geo.geometry.Geometry; import org.elasticsearch.geo.geometry.Line; import org.elasticsearch.geo.geometry.MultiLine; import org.elasticsearch.geo.geometry.MultiPoint; @@ -112,27 +113,39 @@ public void testParsePoint() throws IOException { @Override public void testParseMultiPoint() throws IOException { - int numPoints = randomIntBetween(2, 100); + int numPoints = randomIntBetween(0, 100); List coordinates = new ArrayList<>(numPoints); for (int i = 0; i < numPoints; ++i) { coordinates.add(new Coordinate(GeoTestUtil.nextLongitude(), GeoTestUtil.nextLatitude())); } - Shape[] shapes = new Shape[numPoints]; + List points = new ArrayList<>(numPoints); for (int i = 0; i < numPoints; ++i) { Coordinate c = coordinates.get(i); - shapes[i] = SPATIAL_CONTEXT.makePoint(c.x, c.y); + points.add(new org.elasticsearch.geo.geometry.Point(c.y, c.x)); } - ShapeCollection expected = shapeCollection(shapes); - assertExpected(expected, new MultiPointBuilder(coordinates), true); - List points = new ArrayList<>(numPoints); + Geometry expectedGeom; + MultiPointBuilder actual; + if (numPoints == 0) { + expectedGeom = MultiPoint.EMPTY; + actual = new MultiPointBuilder(); + } else { + expectedGeom = new MultiPoint(points); + actual = new MultiPointBuilder(coordinates); + } + + assertExpected(expectedGeom, actual, false); + assertMalformed(actual); + + assumeTrue("JTS test path cannot handle empty multipoints", numPoints > 1); + Shape[] shapes = new Shape[numPoints]; for (int i = 0; i < numPoints; ++i) { Coordinate c = coordinates.get(i); - points.add(new org.elasticsearch.geo.geometry.Point(c.y, c.x)); + shapes[i] = SPATIAL_CONTEXT.makePoint(c.x, c.y); } - assertExpected(new MultiPoint(points), new MultiPointBuilder(coordinates), false); - assertMalformed(new MultiPointBuilder(coordinates)); + ShapeCollection expected = shapeCollection(shapes); + assertExpected(expected, new MultiPointBuilder(coordinates), true); } private List randomLineStringCoords() { @@ -163,7 +176,7 @@ public void testParseLineString() throws IOException { @Override public void testParseMultiLineString() throws IOException { - int numLineStrings = randomIntBetween(2, 8); + int numLineStrings = randomIntBetween(0, 8); List lineStrings = new ArrayList<>(numLineStrings); MultiLineStringBuilder builder = new MultiLineStringBuilder(); for (int j = 0; j < numLineStrings; ++j) { @@ -173,18 +186,27 @@ public void testParseMultiLineString() throws IOException { builder.linestring(new LineStringBuilder(lsc)); } - MultiLineString expected = GEOMETRY_FACTORY.createMultiLineString( - lineStrings.toArray(new LineString[lineStrings.size()])); - assertExpected(jtsGeom(expected), builder, true); - List lines = new ArrayList<>(lineStrings.size()); for (int j = 0; j < lineStrings.size(); ++j) { Coordinate[] c = lineStrings.get(j).getCoordinates(); lines.add(new Line(Arrays.stream(c).mapToDouble(i->i.y).toArray(), Arrays.stream(c).mapToDouble(i->i.x).toArray())); } - assertExpected(new MultiLine(lines), builder, false); + Geometry expectedGeom; + if (lines.isEmpty()) { + expectedGeom = MultiLine.EMPTY; + } else if (lines.size() == 1) { + expectedGeom = new Line(lines.get(0).getLats(), lines.get(0).getLons()); + } else { + expectedGeom = new MultiLine(lines); + } + assertExpected(expectedGeom, builder, false); assertMalformed(builder); + + MultiLineString expected = GEOMETRY_FACTORY.createMultiLineString( + lineStrings.toArray(new LineString[lineStrings.size()])); + assumeTrue("JTS test path cannot handle empty multilinestrings", numLineStrings > 1); + assertExpected(jtsGeom(expected), builder, true); } @Override @@ -201,7 +223,7 @@ public void testParsePolygon() throws IOException { @Override public void testParseMultiPolygon() throws IOException { - int numPolys = randomIntBetween(2, 8); + int numPolys = randomIntBetween(0, 8); MultiPolygonBuilder builder = new MultiPolygonBuilder(); PolygonBuilder pb; Coordinate[] coordinates; @@ -214,7 +236,7 @@ public void testParseMultiPolygon() throws IOException { shell = GEOMETRY_FACTORY.createLinearRing(coordinates); shapes[i] = GEOMETRY_FACTORY.createPolygon(shell, null); } - + assumeTrue("JTS test path cannot handle empty multipolygon", numPolys > 1); Shape expected = shapeCollection(shapes); assertExpected(expected, builder, true); assertMalformed(builder); @@ -429,7 +451,6 @@ public void testInvalidGeometryType() throws IOException { assertValidException(builder, IllegalArgumentException.class); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37894") @Override public void testParseGeometryCollection() throws IOException { if (rarely()) { From ecbaa388648da7327bf00611a4640ac3475326c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 30 Jan 2019 17:17:54 +0100 Subject: [PATCH 053/100] Remove deprecated Plugin#onModule extension points (#37866) Removes some guice index level extension point marked as @Deprecated since at least 6.0. They served as a signpost for plugin authors upgrading from 2.x but this shouldn't be relevant in 7.0 anymore. --- .../org/elasticsearch/plugins/Plugin.java | 103 ------------------ 1 file changed, 103 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/plugins/Plugin.java b/server/src/main/java/org/elasticsearch/plugins/Plugin.java index faef27207e13a..dcba40b58c21a 100644 --- a/server/src/main/java/org/elasticsearch/plugins/Plugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/Plugin.java @@ -19,10 +19,8 @@ package org.elasticsearch.plugins; -import org.elasticsearch.action.ActionModule; import org.elasticsearch.bootstrap.BootstrapCheck; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; @@ -32,22 +30,15 @@ import org.elasticsearch.common.inject.Module; import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.SettingUpgrader; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.discovery.DiscoveryModule; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexModule; -import org.elasticsearch.indices.analysis.AnalysisModule; -import org.elasticsearch.repositories.RepositoriesModule; -import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; -import org.elasticsearch.search.SearchModule; import org.elasticsearch.threadpool.ExecutorBuilder; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.watcher.ResourceWatcherService; @@ -77,9 +68,6 @@ *

  • {@link SearchPlugin} *
  • {@link ReloadablePlugin} * - *

    In addition to extension points this class also declares some {@code @Deprecated} {@code public final void onModule} methods. These - * methods should cause any extensions of {@linkplain Plugin} that used the pre-5.x style extension syntax to fail to build and point the - * plugin author at the new extension syntax. We hope that these make the process of upgrading a plugin from 2.x to 5.x only mildly painful. */ public abstract class Plugin implements Closeable { @@ -257,95 +245,4 @@ public List> getExecutorBuilders(Settings settings) { public void close() throws IOException { } - - /** - * Old-style guice index level extension point. {@code @Deprecated} and {@code final} to act as a signpost for plugin authors upgrading - * from 2.x. - * - * @deprecated use #onIndexModule instead - */ - @Deprecated - public final void onModule(IndexModule indexModule) {} - - - /** - * Old-style guice settings extension point. {@code @Deprecated} and {@code final} to act as a signpost for plugin authors upgrading - * from 2.x. - * - * @deprecated use #getSettings and #getSettingsFilter instead - */ - @Deprecated - public final void onModule(SettingsModule settingsModule) {} - - /** - * Old-style guice scripting extension point. {@code @Deprecated} and {@code final} to act as a signpost for plugin authors upgrading - * from 2.x. - * - * @deprecated implement {@link ScriptPlugin} instead - */ - @Deprecated - public final void onModule(ScriptModule module) {} - - /** - * Old-style analysis extension point. {@code @Deprecated} and {@code final} to act as a signpost for plugin authors upgrading - * from 2.x. - * - * @deprecated implement {@link AnalysisPlugin} instead - */ - @Deprecated - public final void onModule(AnalysisModule module) {} - - /** - * Old-style action extension point. {@code @Deprecated} and {@code final} to act as a signpost for plugin authors upgrading - * from 2.x. - * - * @deprecated implement {@link ActionPlugin} instead - */ - @Deprecated - public final void onModule(ActionModule module) {} - - /** - * Old-style search extension point. {@code @Deprecated} and {@code final} to act as a signpost for plugin authors upgrading - * from 2.x. - * - * @deprecated implement {@link SearchPlugin} instead - */ - @Deprecated - public final void onModule(SearchModule module) {} - - /** - * Old-style network extension point. {@code @Deprecated} and {@code final} to act as a signpost for plugin authors upgrading - * from 2.x. - * - * @deprecated implement {@link NetworkPlugin} instead - */ - @Deprecated - public final void onModule(NetworkModule module) {} - - /** - * Old-style snapshot/restore extension point. {@code @Deprecated} and {@code final} to act as a signpost for plugin authors upgrading - * from 2.x. - * - * @deprecated implement {@link RepositoryPlugin} instead - */ - @Deprecated - public final void onModule(RepositoriesModule module) {} - - /** - * Old-style cluster extension point. {@code @Deprecated} and {@code final} to act as a signpost for plugin authors upgrading - * from 2.x. - * - * @deprecated implement {@link ClusterPlugin} instead - */ - @Deprecated - public final void onModule(ClusterModule module) {} - - /** - * Old-style discovery extension point. {@code @Deprecated} and {@code final} to act as a signpost for plugin authors upgrading - * from 2.x. - * - * @deprecated implement {@link DiscoveryPlugin} instead - */ - @Deprecated - public final void onModule(DiscoveryModule module) {} } From 2f7776c8b78c40a538827a195014d52c1d1e5ccf Mon Sep 17 00:00:00 2001 From: David Roberts Date: Wed, 30 Jan 2019 16:26:28 +0000 Subject: [PATCH 054/100] Switch default time format for ingest from Joda to Java for v7 (#37934) Date formats with and without the "8" prefix are now all treated as Java time formats, so that ingest does the same as mappings in this respect. --- .../ingest/common/DateFormat.java | 37 +++++++------------ .../ingest/common/DateIndexNameProcessor.java | 2 +- .../common/DateIndexNameProcessorTests.java | 6 +-- .../ingest/common/DateProcessorTests.java | 21 +++++------ .../test/ingest/20_combine_processors.yml | 8 ++-- 5 files changed, 31 insertions(+), 43 deletions(-) diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java index 220091c4baadc..a15bc5049801f 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java @@ -19,7 +19,6 @@ package org.elasticsearch.ingest.common; -import org.elasticsearch.common.joda.Joda; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.DateFormatters; import org.elasticsearch.common.time.DateUtils; @@ -73,30 +72,22 @@ private long parseMillis(String date) { Java { @Override Function getFunction(String format, DateTimeZone timezone, Locale locale) { - // in case you are wondering why we do not call 'DateFormatter.forPattern(format)' for all cases here, but only for the - // non java time case: - // When the joda date formatter parses a date then a year is always set, so that no fallback can be used, like - // done in the JodaDateFormatter.withYear() code below - // This means that we leave the existing parsing logic in place, but will fall back to the new java date parsing logic, if an - // "8" is prepended to the date format string - int year = LocalDate.now(ZoneOffset.UTC).getYear(); + + // support the 6.x BWC compatible way of parsing java 8 dates if (format.startsWith("8")) { - DateFormatter formatter = DateFormatter.forPattern(format) - .withLocale(locale) - .withZone(DateUtils.dateTimeZoneToZoneId(timezone)); - return text -> { - ZonedDateTime defaultZonedDateTime = Instant.EPOCH.atZone(ZoneOffset.UTC).withYear(year); - TemporalAccessor accessor = formatter.parse(text); - long millis = DateFormatters.toZonedDateTime(accessor, defaultZonedDateTime).toInstant().toEpochMilli(); - return new DateTime(millis, timezone); - }; - } else { - DateFormatter formatter = Joda.forPattern(format) - .withYear(year) - .withZone(DateUtils.dateTimeZoneToZoneId(timezone)) - .withLocale(locale); - return text -> new DateTime(formatter.parseMillis(text), timezone); + format = format.substring(1); } + + int year = LocalDate.now(ZoneOffset.UTC).getYear(); + DateFormatter formatter = DateFormatter.forPattern(format) + .withLocale(locale) + .withZone(DateUtils.dateTimeZoneToZoneId(timezone)); + return text -> { + ZonedDateTime defaultZonedDateTime = Instant.EPOCH.atZone(ZoneOffset.UTC).withYear(year); + TemporalAccessor accessor = formatter.parse(text); + long millis = DateFormatters.toZonedDateTime(accessor, defaultZonedDateTime).toInstant().toEpochMilli(); + return new DateTime(millis, timezone); + }; } }; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java index 4a88f15b6410d..ca429375f792e 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java @@ -157,7 +157,7 @@ public DateIndexNameProcessor create(Map registry, St } List dateFormatStrings = ConfigurationUtils.readOptionalList(TYPE, tag, config, "date_formats"); if (dateFormatStrings == null) { - dateFormatStrings = Collections.singletonList("yyyy-MM-dd'T'HH:mm:ss.SSSZ"); + dateFormatStrings = Collections.singletonList("yyyy-MM-dd'T'HH:mm:ss.SSSXX"); } List> dateFormats = new ArrayList<>(dateFormatStrings.size()); for (String format : dateFormatStrings) { diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java index 6555628f1da15..760e48f31ff29 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java @@ -34,8 +34,8 @@ public class DateIndexNameProcessorTests extends ESTestCase { - public void testJodaPattern() throws Exception { - Function function = DateFormat.Java.getFunction("yyyy-MM-dd'T'HH:mm:ss.SSSZ", DateTimeZone.UTC, Locale.ROOT); + public void testJavaPattern() throws Exception { + Function function = DateFormat.Java.getFunction("yyyy-MM-dd'T'HH:mm:ss.SSSXX", DateTimeZone.UTC, Locale.ROOT); DateIndexNameProcessor processor = createProcessor("_field", Collections.singletonList(function), DateTimeZone.UTC, "events-", "y", "yyyyMMdd"); IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, @@ -82,7 +82,7 @@ public void testUnix()throws Exception { public void testTemplatedFields() throws Exception { String indexNamePrefix = randomAlphaOfLength(10); String dateRounding = randomFrom("y", "M", "w", "d", "h", "m", "s"); - String indexNameFormat = randomFrom("yyyy-MM-dd'T'HH:mm:ss.SSSZ", "yyyyMMdd", "MM/dd/yyyy"); + String indexNameFormat = randomFrom("yyyy-MM-dd'T'HH:mm:ss.SSSXX", "yyyyMMdd", "MM/dd/yyyy"); String date = Integer.toString(randomInt()); Function dateTimeFunction = DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java index 0f31143c43d0e..6157e3e9e50f9 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java @@ -49,10 +49,11 @@ private TemplateScript.Factory templatize(ZoneId timezone) { String id = timezone.equals(ZoneOffset.UTC) ? "UTC" : timezone.getId(); return new TestTemplateService.MockTemplateScript.Factory(id); } - public void testJodaPattern() { + + public void testJavaPattern() { DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), templatize(ZoneId.of("Europe/Amsterdam")), templatize(Locale.ENGLISH), - "date_as_string", Collections.singletonList("yyyy dd MM hh:mm:ss"), "date_as_date"); + "date_as_string", Collections.singletonList("yyyy dd MM HH:mm:ss"), "date_as_date"); Map document = new HashMap<>(); document.put("date_as_string", "2010 12 06 11:05:15"); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); @@ -60,7 +61,7 @@ public void testJodaPattern() { assertThat(ingestDocument.getFieldValue("date_as_date", String.class), equalTo("2010-06-12T11:05:15.000+02:00")); } - public void testJodaPatternMultipleFormats() { + public void testJavaPatternMultipleFormats() { List matchFormats = new ArrayList<>(); matchFormats.add("yyyy dd MM"); matchFormats.add("dd/MM/yyyy"); @@ -98,7 +99,7 @@ public void testJodaPatternMultipleFormats() { } } - public void testInvalidJodaPattern() { + public void testInvalidJavaPattern() { try { DateProcessor processor = new DateProcessor(randomAlphaOfLength(10), templatize(ZoneOffset.UTC), templatize(randomLocale(random())), @@ -109,16 +110,14 @@ public void testInvalidJodaPattern() { fail("date processor execution should have failed"); } catch(IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("unable to parse date [2010]")); - assertThat(e.getCause().getMessage(), equalTo("Invalid format: [invalid pattern]: Illegal pattern component: i")); + assertThat(e.getCause().getMessage(), equalTo("Invalid format: [invalid pattern]: Unknown pattern letter: i")); } } - public void testJodaPatternLocale() { - //TODO investigate if this is a bug in Joda - assumeFalse("Can't run in a FIPS JVM, Joda parse date error", inFipsJvm()); - DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), + public void testJavaPatternLocale() { + DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), templatize(ZoneId.of("Europe/Amsterdam")), templatize(Locale.ITALIAN), - "date_as_string", Collections.singletonList("yyyy dd MMM"), "date_as_date"); + "date_as_string", Collections.singletonList("yyyy dd MMMM"), "date_as_date"); Map document = new HashMap<>(); document.put("date_as_string", "2010 12 giugno"); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); @@ -126,7 +125,7 @@ public void testJodaPatternLocale() { assertThat(ingestDocument.getFieldValue("date_as_date", String.class), equalTo("2010-06-12T00:00:00.000+02:00")); } - public void testJodaPatternDefaultYear() { + public void testJavaPatternDefaultYear() { String format = randomFrom("dd/MM", "8dd/MM"); DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), templatize(ZoneId.of("Europe/Amsterdam")), templatize(Locale.ENGLISH), diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/20_combine_processors.yml b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/20_combine_processors.yml index f44e6ae5753a0..c121d542c86b1 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/20_combine_processors.yml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/20_combine_processors.yml @@ -1,8 +1,8 @@ --- -"Test logging": +"Test with date processor": - skip: version: " - 6.9.99" - reason: pre-7.0.0 will send no warnings + reason: pre-7.0.0 requires the 8 prefix for Java time formats, so would treat the format in this test as a Joda time format features: "warnings" - do: @@ -33,7 +33,7 @@ "date" : { "field" : "timestamp", "target_field" : "timestamp", - "formats" : ["dd/MMM/YYYY:HH:mm:ss Z"] + "formats" : ["dd/MMM/yyyy:HH:mm:ss xx"] } }, { @@ -46,8 +46,6 @@ - match: { acknowledged: true } - do: - warnings: - - "Use of 'Y' (year-of-era) will change to 'y' in the next major version of Elasticsearch. Prefix your date format with '8' to use the new specifier." index: index: test type: test From 21e392e95e6ab5b293a46f7a2af84c0fa4778029 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Wed, 30 Jan 2019 16:32:58 +0000 Subject: [PATCH 055/100] Removes typed calls from YAML REST tests (#37611) This PR attempts to remove all typed calls from our YAML REST tests. The PR adds include_type_name: false to create index requests that use a mapping and also to put mapping requests. It also removes _type from index requests where they haven't already been removed. The PR ignores tests named *_with_types.yml since this are specifically testing typed API behaviour. The change also includes changing the test harness to add the type _doc to index, update, get and bulk requests that do not specify the document type when the test is running against a mixed 7.x/6.x cluster. --- .../test/stats/20_empty_bucket.yml | 14 +- .../test/stats/30_single_value_field.yml | 31 +- .../test/stats/40_multi_value_field.yml | 31 +- .../test/indices.analyze/10_analyze.yml | 12 +- .../indices/validate_query/10_synonyms.yml | 13 +- .../test/search.query/10_match.yml | 14 +- .../test/search.query/20_ngram_search.yml | 28 +- .../search.query/30_ngram_highligthing.yml | 25 +- .../test/search.query/40_query_string.yml | 13 +- .../search.query/50_queries_with_synonyms.yml | 35 +- .../test/search.query/60_synonym_graph.yml | 16 +- .../test/search.suggest/20_phrase.yml | 32 +- .../test/search.suggest/30_synonyms.yml | 11 +- .../test/termvectors/10_payloads.yml | 14 +- .../ingest/100_date_index_name_processor.yml | 1 - .../rest-api-spec/test/ingest/110_sort.yml | 2 - .../rest-api-spec/test/ingest/120_grok.yml | 6 - .../test/ingest/130_escape_dot.yml | 2 - .../rest-api-spec/test/ingest/140_json.yml | 2 - .../rest-api-spec/test/ingest/150_kv.yml | 2 - .../test/ingest/160_urldecode.yml | 2 - .../rest-api-spec/test/ingest/170_version.yml | 2 - .../test/ingest/180_bytes_processor.yml | 3 - .../test/ingest/190_script_processor.yml | 12 - .../test/ingest/200_default_pipeline.yml | 27 +- .../test/ingest/200_dissect_processor.yml | 3 - .../test/ingest/210_conditional_processor.yml | 5 - .../test/ingest/210_pipeline_processor.yml | 3 - .../test/ingest/220_drop_processor.yml | 6 - .../test/ingest/30_date_processor.yml | 3 - .../rest-api-spec/test/ingest/40_mutate.yml | 5 - .../test/ingest/50_on_failure.yml | 6 - .../rest-api-spec/test/ingest/60_fail.yml | 4 - .../rest-api-spec/test/ingest/70_bulk.yml | 9 - .../test/ingest_geoip/20_geoip_processor.yml | 12 - .../20_useragent_processor.yml | 4 - .../test/ingest-useragent/30_custom_regex.yml | 2 - .../test/lang_expression/20_search.yml | 10 +- .../20_render_search_template.yml | 2 - .../lang_mustache/25_custom_functions.yml | 7 +- .../test/lang_mustache/30_search_template.yml | 9 - .../50_multi_search_template.yml | 5 - .../rest-api-spec/test/painless/15_update.yml | 19 +- .../test/painless/20_scriptfield.yml | 26 +- .../test/painless/25_script_upsert.yml | 10 - .../rest-api-spec/test/painless/30_search.yml | 17 +- .../test/painless/40_disabled.yml | 2 - .../test/painless/50_script_doc_values.yml | 65 ++-- .../painless/60_script_doc_values_binary.yml | 12 +- .../painless/70_execute_painless_scripts.yml | 16 +- .../test/painless/70_mov_fn_agg.yml | 24 +- .../test/painless/80_script_score.yml | 29 +- .../painless/90_interval_query_filter.yml | 20 +- .../test/dense-vector/10_indexing.yml | 9 +- .../test/rank_feature/10_basic.yml | 18 +- .../test/rank_features/10_basic.yml | 12 +- .../test/scaled_float/10_basic.yml | 15 +- .../test/sparse-vector/10_indexing.yml | 9 +- .../rest-api-spec/test/11_parent_child.yml | 14 +- .../rest-api-spec/test/20_parent_join.yml | 30 +- .../resources/rest-api-spec/test/10_basic.yml | 13 +- .../rest-api-spec/test/rank_eval/10_basic.yml | 20 +- .../rest-api-spec/test/rank_eval/20_dcg.yml | 12 +- .../test/rank_eval/30_failures.yml | 1 - .../test/rank_eval/40_rank_eval_templated.yml | 6 - .../test/delete_by_query/10_basic.yml | 13 - .../test/delete_by_query/20_validation.yml | 3 - .../test/delete_by_query/40_versioning.yml | 2 - .../50_wait_for_active_shards.yml | 1 - .../test/delete_by_query/70_throttle.yml | 12 - .../test/delete_by_query/80_slices.yml | 18 - .../rest-api-spec/test/reindex/10_basic.yml | 12 - .../test/reindex/20_validation.yml | 4 - .../test/reindex/25_no_auto_create.yml | 3 - .../rest-api-spec/test/reindex/30_search.yml | 4 - .../test/reindex/35_search_failures.yml | 1 - .../test/reindex/40_versioning.yml | 12 - .../rest-api-spec/test/reindex/50_routing.yml | 4 - .../reindex/60_wait_for_active_shards.yml | 2 - .../test/reindex/70_throttle.yml | 12 - .../rest-api-spec/test/reindex/80_slices.yml | 18 - .../test/reindex/85_scripting.yml | 23 -- .../rest-api-spec/test/reindex/90_remote.yml | 14 - .../test/reindex/95_parent_join.yml | 3 - .../test/update_by_query/10_basic.yml | 15 - .../test/update_by_query/20_validation.yml | 4 - .../test/update_by_query/30_new_fields.yml | 1 - .../update_by_query/35_search_failure.yml | 1 - .../test/update_by_query/40_versioning.yml | 4 - .../test/update_by_query/50_consistency.yml | 2 - .../test/update_by_query/60_throttle.yml | 12 - .../test/update_by_query/70_slices.yml | 18 - .../test/update_by_query/80_scripting.yml | 23 -- .../test/repository_url/10_basic.yml | 9 +- .../test/analysis_icu/20_search.yml | 11 +- .../test/analysis_nori/20_search.yml | 1 - .../test/analysis_nori/20_search.yml | 1 - .../test/analysis_phonetic/40_search.yml | 11 +- .../test/analysis_smartcn/20_search.yml | 11 +- .../test/analysis_stempel/20_search.yml | 11 +- .../test/analysis_ukrainian/20_search.yml | 11 +- .../test/custom-suggester/20_suggest.yml | 1 - .../test/painless_whitelist/20_whitelist.yml | 1 - .../test/painless_whitelist/30_static.yml | 1 - .../test/painless_whitelist/40_instance.yml | 1 - .../test/example-rescore/20_score.yml | 2 - .../test/script_expert_scoring/20_score.yml | 3 - .../20_attachment_processor.yml | 12 - .../ingest_attachment/30_files_supported.yml | 5 - .../test/mapper_annotatedtext/10_basic.yml | 15 +- .../test/mapper_murmur3/10_basic.yml | 8 +- .../test/mapper_size/10_basic.yml | 6 +- .../20_repository_permanent_credentials.yml | 7 - .../30_repository_temporary_credentials.yml | 7 - .../40_repository_ec2_credentials.yml | 7 - .../50_repository_ecs_credentials.yml | 7 - .../test/store_smb/15_index_creation.yml | 4 +- .../test/cat.aliases/10_basic.yml | 8 +- .../rest-api-spec/test/cat.count/10_basic.yml | 2 - .../test/cat.fielddata/10_basic.yml | 11 +- .../test/cat.recovery/10_basic.yml | 1 - .../test/cat.segments/10_basic.yml | 5 - .../test/cat.shards/10_basic.yml | 1 - .../test/cluster.state/20_filtering.yml | 4 - .../rest-api-spec/test/count/10_basic.yml | 1 - .../test/count/20_query_string.yml | 9 +- .../rest-api-spec/test/exists/10_basic.yml | 1 - .../rest-api-spec/test/exists/40_routing.yml | 1 - .../test/exists/60_realtime_refresh.yml | 3 +- .../rest-api-spec/test/exists/70_defaults.yml | 1 - .../rest-api-spec/test/explain/10_basic.yml | 2 - .../test/explain/20_source_filtering.yml | 1 - .../test/explain/30_query_string.yml | 9 +- .../test/field_caps/10_basic.yml | 168 ++++----- .../test/get_source/15_default_values.yml | 1 - .../test/get_source/40_routing.yml | 2 - .../test/get_source/60_realtime_refresh.yml | 2 - .../test/get_source/70_source_filtering.yml | 1 - .../test/get_source/85_source_missing.yml | 5 +- .../test/indices.analyze/10_analyze.yml | 10 +- .../test/indices.get/10_basic.yml | 41 +-- .../test/indices.get/11_basic_with_types.yml | 77 ++++ .../30_missing_type.yml | 2 +- .../indices.get_mapping/20_missing_type.yml | 2 +- .../test/indices.get_mapping/40_aliases.yml | 3 +- .../50_wildcard_expansion.yml | 20 +- .../test/indices.put_mapping/10_basic.yml | 4 +- .../test/indices.shrink/10_basic.yml | 7 +- .../test/indices.shrink/20_source_mapping.yml | 11 +- .../test/indices.sort/10_basic.yml | 18 +- .../test/indices.split/10_basic.yml | 24 +- .../test/indices.split/20_source_mapping.yml | 11 +- .../test/indices.stats/10_index.yml | 2 - .../test/indices.stats/11_metric.yml | 2 - .../test/indices.stats/12_level.yml | 3 - .../test/indices.stats/13_fields.yml | 30 +- .../test/indices.stats/14_groups.yml | 2 - .../test/indices.stats/15_types.yml | 1 - .../test/indices.stats/20_translog.yml | 2 - .../20_query_string.yml | 12 +- .../rest-api-spec/test/mget/10_basic.yml | 1 - .../test/mget/12_non_existent_index.yml | 1 - .../test/mget/13_missing_metadata.yml | 1 - .../rest-api-spec/test/mget/15_ids.yml | 2 - .../test/mget/17_default_index.yml | 1 - .../test/mget/20_stored_fields.yml | 18 +- .../rest-api-spec/test/mget/40_routing.yml | 1 - .../test/mget/60_realtime_refresh.yml | 1 - .../test/mget/70_source_filtering.yml | 2 - .../rest-api-spec/test/mget/80_deprecated.yml | 2 - .../rest-api-spec/test/mlt/10_basic.yml | 14 +- .../rest-api-spec/test/mlt/20_docs.yml | 7 +- .../rest-api-spec/test/mlt/30_unlike.yml | 7 +- .../rest-api-spec/test/msearch/10_basic.yml | 7 - .../test/msearch/20_typed_keys.yml | 58 +-- .../test/mtermvectors/10_basic.yml | 11 +- .../test/mtermvectors/20_deprecated.yml | 11 +- .../rest-api-spec/test/range/10_basic.yml | 47 +-- .../rest-api-spec/test/scroll/10_basic.yml | 14 - .../rest-api-spec/test/scroll/11_clear.yml | 5 +- .../rest-api-spec/test/scroll/12_slices.yml | 5 - .../test/scroll/20_keep_alive.yml | 2 - .../search.aggregation/100_avg_metric.yml | 21 +- .../test/search.aggregation/10_histogram.yml | 94 +++-- .../search.aggregation/110_max_metric.yml | 20 +- .../search.aggregation/120_min_metric.yml | 21 +- .../search.aggregation/130_sum_metric.yml | 21 +- .../140_value_count_metric.yml | 20 +- .../search.aggregation/150_stats_metric.yml | 22 +- .../160_extended_stats_metric.yml | 20 +- .../170_cardinality_metric.yml | 22 +- .../180_percentiles_tdigest_metric.yml | 23 +- .../190_percentiles_hdr_metric.yml | 23 +- .../200_top_hits_metric.yml | 10 +- .../test/search.aggregation/20_terms.yml | 77 ++-- .../search.aggregation/220_filters_bucket.yml | 21 +- .../test/search.aggregation/230_composite.yml | 32 +- .../search.aggregation/240_max_buckets.yml | 20 +- .../search.aggregation/260_weighted_avg.yml | 21 +- .../270_median_absolute_deviation_metric.yml | 19 +- .../test/search.aggregation/30_sig_terms.yml | 41 +-- .../test/search.aggregation/40_range.yml | 43 +-- .../test/search.aggregation/50_filter.yml | 21 +- .../51_filter_with_types.yml | 57 +++ .../70_adjacency_matrix.yml | 12 +- .../test/search.aggregation/80_typed_keys.yml | 17 +- .../test/search.aggregation/90_sig_text.yml | 53 +-- .../test/search.highlight/10_unified.yml | 23 +- .../test/search.highlight/20_fvh.yml | 17 +- .../30_max_analyzed_offset.yml | 15 +- .../test/search.inner_hits/10_basic.yml | 22 +- .../test/search/100_stored_fields.yml | 2 - .../test/search/10_source_filtering.yml | 9 +- .../test/search/110_field_collapsing.yml | 44 +-- .../search/115_multiple_field_collapsing.yml | 26 +- .../test/search/120_batch_reduce_size.yml | 13 +- .../search/140_pre_filter_search_shards.yml | 41 +-- .../search/150_rewrite_on_coordinator.yml | 14 +- .../test/search/160_exists_query.yml | 335 +++++++++--------- .../test/search/170_terms_query.yml | 20 +- .../search/180_locale_dependent_mapping.yml | 16 +- .../test/search/190_index_prefix_search.yml | 16 +- .../test/search/200_ignore_malformed.yml | 21 +- .../test/search/200_index_phrase_search.yml | 13 +- .../test/search/20_default_values.yml | 6 +- .../test/search/210_rescore_explain.yml | 6 +- .../test/search/220_total_hits_object.yml | 9 - .../test/search/230_interval_query.yml | 20 +- .../rest-api-spec/test/search/30_limits.yml | 5 +- .../test/search/60_query_string.yml | 9 +- .../test/search/70_response_filtering.yml | 6 +- .../test/search/90_search_after.yml | 9 +- .../rest-api-spec/test/search/issue4895.yml | 4 - .../rest-api-spec/test/search/issue9606.yml | 3 - .../test/search_shards/10_basic.yml | 8 +- .../rest-api-spec/test/suggest/10_basic.yml | 2 - .../test/suggest/20_completion.yml | 52 ++- .../rest-api-spec/test/suggest/30_context.yml | 87 ++--- .../test/suggest/40_typed_keys.yml | 21 +- .../50_completion_with_multi_fields.yml | 82 ++--- .../test/termvectors/10_basic.yml | 11 +- .../test/termvectors/20_issue7121.yml | 11 +- .../test/termvectors/30_realtime.yml | 1 - .../test/update/13_legacy_doc.yml | 6 +- .../yaml/ClientYamlTestExecutionContext.java | 40 +++ 245 files changed, 1433 insertions(+), 2342 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/11_basic_with_types.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/51_filter_with_types.yml diff --git a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/20_empty_bucket.yml b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/20_empty_bucket.yml index 3e124a6b26288..09896371574ad 100644 --- a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/20_empty_bucket.yml +++ b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/20_empty_bucket.yml @@ -2,29 +2,27 @@ "Empty Bucket Aggregation": - do: indices.create: + include_type_name: false index: empty_bucket_idx body: settings: number_of_shards: "3" mappings: - test: - "properties": - "value": - "type": "integer" - "val1": - "type": "double" + "properties": + "value": + "type": "integer" + "val1": + "type": "double" - do: index: index: empty_bucket_idx - type: test id: 1 body: { "value": 0, "val1": 3.1 } - do: index: index: empty_bucket_idx - type: test id: 2 body: { "value": 2, "val1": -3.1 } diff --git a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/30_single_value_field.yml b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/30_single_value_field.yml index 2630c88fd12bc..630211feb08e0 100644 --- a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/30_single_value_field.yml +++ b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/30_single_value_field.yml @@ -3,20 +3,20 @@ setup: - do: indices.create: + include_type_name: false index: test body: settings: number_of_shards: 3 number_of_routing_shards: 3 mappings: - test: - "properties": - "val1": - "type": "double" - "val2": - "type": "double" - "val3": - "type": "double" + "properties": + "val1": + "type": "double" + "val2": + "type": "double" + "val3": + "type": "double" - do: indices.create: @@ -28,91 +28,76 @@ setup: - do: index: index: test - type: test id: 1 body: { "val1": 1.9, "val2": 3.1, "val3": 2.3 } - do: index: index: test - type: test id: 2 body: { "val1": -5.2, "val2": -3.4, "val3": 2.3} - do: index: index: test - type: test id: 3 body: { "val1": -5.2, "val3": 2.3} - do: index: index: test - type: test id: 4 body: { "val1": 18.3, "val2": 104.4, "val3": 2.3} - do: index: index: test - type: test id: 5 body: { "val1": -53.2, "val2": -322.4, "val3": 2.3} - do: index: index: test - type: test id: 6 body: { "val1": -578.9, "val2": 69.9, "val3": 2.3} - do: index: index: test - type: test id: 7 body: { "val1": 16.2, "val2": 17.2, "val3": 2.3} - do: index: index: test - type: test id: 8 body: { "val1": -4222.63, "val2": 316.44, "val3": 2.3} - do: index: index: test - type: test id: 9 body: { "val1": -59999.55, "val2": -3163.4, "val3": 2.3} - do: index: index: test - type: test id: 10 body: { "val1": 782.7, "val2": 789.7, "val3": 2.3} - do: index: index: test - type: test id: 11 body: { "val1": -1.2, "val2": 6.3, "val3": 2.3} - do: index: index: test - type: test id: 12 body: { "val1": 0, "val2": 1.11, "val3": 2.3} - do: index: index: test - type: test id: 13 body: { "val1": 0.1, "val2": 0.92, "val3": 2.3} - do: index: index: test - type: test id: 14 body: { "val1": 0.12, "val2": -82.4, "val3": 2.3} - do: index: index: test - type: test id: 15 body: { "val1": 98.2, "val2": 32.4, "val3": 2.3} diff --git a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/40_multi_value_field.yml b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/40_multi_value_field.yml index 0b3b1b28755ac..bc9634d412694 100644 --- a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/40_multi_value_field.yml +++ b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/40_multi_value_field.yml @@ -3,20 +3,20 @@ setup: - do: indices.create: + include_type_name: false index: test body: settings: number_of_shards: 3 number_of_routing_shards: 3 mappings: - test: - "properties": - "val1": - "type": "double" - "val2": - "type": "double" - "val3": - "type": "double" + "properties": + "val1": + "type": "double" + "val2": + "type": "double" + "val3": + "type": "double" - do: indices.create: @@ -28,91 +28,76 @@ setup: - do: index: index: test - type: test id: 1 body: { "val1": 1.9, "val2": 3.1, "val3": 2.3, "vals" : [1.9, 16.143] } - do: index: index: test - type: test id: 2 body: { "val1": -5.2, "val2": -3.4, "val3": 2.3, "vals" : [155, 16.23]} - do: index: index: test - type: test id: 3 body: { "val1": -5.2, "val3": 2.3, "vals" : [-455, -32.32]} - do: index: index: test - type: test id: 4 body: { "val1": 18.3, "val2": 104.4, "val3": 2.3, "vals" : [0.14, 92.1]} - do: index: index: test - type: test id: 5 body: { "val1": -53.2, "val2": -322.4, "val3": 2.3, "vals" : [16, 16]} - do: index: index: test - type: test id: 6 body: { "val1": -578.9, "val2": 69.9, "val3": 2.3} - do: index: index: test - type: test id: 7 body: { "val1": 16.2, "val2": 17.2, "val3": 2.3, "vals" : [1234.3, -3433]} - do: index: index: test - type: test id: 8 body: { "val1": -4222.63, "val2": 316.44, "val3": 2.3, "vals" : [177.2, -93.333]} - do: index: index: test - type: test id: 9 body: { "val1": -59999.55, "val2": -3163.4, "val3": 2.3, "vals" : [-29.9, 163.0]} - do: index: index: test - type: test id: 10 body: { "val1": 782.7, "val2": 789.7, "val3": 2.3, "vals" : [-0.2, 1343.3]} - do: index: index: test - type: test id: 11 body: { "val1": -1.2, "val2": 6.3, "val3": 2.3, "vals" : [15.3, 16.9]} - do: index: index: test - type: test id: 12 body: { "val1": 0, "val2": 1.11, "val3": 2.3, "vals" : [-644.4, -644.4]} - do: index: index: test - type: test id: 13 body: { "val1": 0.1, "val2": 0.92, "val3": 2.3, "vals" : [73.2, 0.12]} - do: index: index: test - type: test id: 14 body: { "val1": 0.12, "val2": -82.4, "val3": 2.3, "vals" : [-0.001, 1295.3]} - do: index: index: test - type: test id: 15 body: { "val1": 98.2, "val2": 32.4, "val3": 2.3, "vals" : [15.5, 16.5]} diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/indices.analyze/10_analyze.yml b/modules/analysis-common/src/test/resources/rest-api-spec/test/indices.analyze/10_analyze.yml index 936736e93de93..3ceb26b2bac21 100644 --- a/modules/analysis-common/src/test/resources/rest-api-spec/test/indices.analyze/10_analyze.yml +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/indices.analyze/10_analyze.yml @@ -23,6 +23,7 @@ - do: indices.create: + include_type_name: false index: test_deprecated_htmlstrip body: settings: @@ -33,18 +34,16 @@ tokenizer: keyword char_filter: ["htmlStrip"] mappings: - type: - properties: - name: - type: text - analyzer: my_htmlStripWithCharfilter + properties: + name: + type: text + analyzer: my_htmlStripWithCharfilter - do: warnings: - 'The [htmpStrip] char filter name is deprecated and will be removed in a future version. Please change the filter name to [html_strip] instead.' index: index: test_deprecated_htmlstrip - type: type id: 1 body: { "name": "foo bar" } @@ -53,7 +52,6 @@ - 'The [htmpStrip] char filter name is deprecated and will be removed in a future version. Please change the filter name to [html_strip] instead.' index: index: test_deprecated_htmlstrip - type: type id: 2 body: { "name": "foo baz" } diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/indices/validate_query/10_synonyms.yml b/modules/analysis-common/src/test/resources/rest-api-spec/test/indices/validate_query/10_synonyms.yml index a0ef4463f21dc..7509d9667ef1b 100644 --- a/modules/analysis-common/src/test/resources/rest-api-spec/test/indices/validate_query/10_synonyms.yml +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/indices/validate_query/10_synonyms.yml @@ -2,6 +2,7 @@ "validate query with synonyms": - do: indices.create: + include_type_name: false index: test body: settings: @@ -16,11 +17,10 @@ tokenizer: standard filter: [ syns ] mappings: - test: - properties: - field: - type: text - analyzer: syns + properties: + field: + type: text + analyzer: syns - do: indices.validate_query: @@ -77,6 +77,3 @@ - is_true: valid - length: { explanations: 1 } - match: { explanations.0.explanation: "field:\"foo (one* two*)\"" } - - - diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/10_match.yml b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/10_match.yml index 2f97450012dc7..78e2507e462b8 100644 --- a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/10_match.yml +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/10_match.yml @@ -5,6 +5,7 @@ # versions in the same position. - do: indices.create: + include_type_name: false index: test body: settings: @@ -24,17 +25,15 @@ type: unique only_on_same_position: true mappings: - doc: - properties: - text: - type: text - analyzer: index - search_analyzer: search + properties: + text: + type: text + analyzer: index + search_analyzer: search - do: index: index: test - type: doc id: 1 body: { "text": "the fox runs across the street" } refresh: true @@ -53,7 +52,6 @@ - do: index: index: test - type: doc id: 2 body: { "text": "run fox run" } refresh: true diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/20_ngram_search.yml b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/20_ngram_search.yml index 8b5fd16904c24..f480045689056 100644 --- a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/20_ngram_search.yml +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/20_ngram_search.yml @@ -1,6 +1,7 @@ "ngram search": - do: indices.create: + include_type_name: false index: test body: settings: @@ -17,16 +18,14 @@ min: 2, max: 2 mappings: - doc: - properties: - text: - type: text - analyzer: my_analyzer + properties: + text: + type: text + analyzer: my_analyzer - do: index: index: test - type: doc id: 1 body: { "text": "foo bar baz" } refresh: true @@ -45,6 +44,7 @@ "testNGramCopyField": - do: indices.create: + include_type_name: false index: test body: settings: @@ -62,19 +62,17 @@ max: 10 token_chars: [] mappings: - doc: - properties: - origin: - type: text - copy_to: meta - meta: - type: text - analyzer: my_ngram_analyzer + properties: + origin: + type: text + copy_to: meta + meta: + type: text + analyzer: my_ngram_analyzer - do: index: index: test - type: doc id: 1 body: { "origin": "C.A1234.5678" } refresh: true diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/30_ngram_highligthing.yml b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/30_ngram_highligthing.yml index a69faecf2cda0..943f1d6ae5767 100644 --- a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/30_ngram_highligthing.yml +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/30_ngram_highligthing.yml @@ -1,6 +1,7 @@ "ngram highlighting": - do: indices.create: + include_type_name: false index: test body: settings: @@ -28,23 +29,21 @@ name_search_analyzer: tokenizer: whitespace mappings: - doc: - properties: - name: - type: text - term_vector: with_positions_offsets - analyzer: name_index_analyzer - search_analyzer: name_search_analyzer - name2: - type: text - term_vector: with_positions_offsets - analyzer: name2_index_analyzer - search_analyzer: name_search_analyzer + properties: + name: + type: text + term_vector: with_positions_offsets + analyzer: name_index_analyzer + search_analyzer: name_search_analyzer + name2: + type: text + term_vector: with_positions_offsets + analyzer: name2_index_analyzer + search_analyzer: name_search_analyzer - do: index: index: test - type: doc id: 1 refresh: true body: diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/40_query_string.yml b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/40_query_string.yml index 3a6f02c17e5b9..efe5d04012d57 100644 --- a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/40_query_string.yml +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/40_query_string.yml @@ -2,20 +2,19 @@ "Test query string with snowball": - do: indices.create: + include_type_name: false index: test body: mappings: - test: - properties: - field: - type: text - number: - type: integer + properties: + field: + type: text + number: + type: integer - do: index: index: test - type: test id: 1 body: { field: foo bar} diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/50_queries_with_synonyms.yml b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/50_queries_with_synonyms.yml index 874b4852244cc..5a4af7c9a344a 100644 --- a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/50_queries_with_synonyms.yml +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/50_queries_with_synonyms.yml @@ -2,6 +2,7 @@ "Test common terms query with stacked tokens": - do: indices.create: + include_type_name: false index: test body: settings: @@ -15,19 +16,17 @@ tokenizer: standard filter: [ "syns" ] mappings: - test: - properties: - field1: - type: text - analyzer: syns - field2: - type: text - analyzer: syns + properties: + field1: + type: text + analyzer: syns + field2: + type: text + analyzer: syns - do: index: index: test - type: test id: 3 body: field1: quick lazy huge brown pidgin @@ -36,7 +35,6 @@ - do: index: index: test - type: test id: 1 body: field1: the quick brown fox @@ -44,7 +42,6 @@ - do: index: index: test - type: test id: 2 body: field1: the quick lazy huge brown fox jumps over the tree @@ -223,6 +220,7 @@ "Test match query with synonyms - see #3881 for extensive description of the issue": - do: indices.create: + include_type_name: false index: test body: settings: @@ -242,17 +240,15 @@ tokenizer: standard filter: [ lowercase, synonym ] mappings: - test: - properties: - text: - type: text - analyzer: index - search_analyzer: search + properties: + text: + type: text + analyzer: index + search_analyzer: search - do: index: index: test - type: test id: 1 body: text: quick brown fox @@ -294,7 +290,6 @@ - do: index: index: test - type: test id: 2 body: text: fast brown fox @@ -321,5 +316,3 @@ query: quick brown operator: and - match: { hits.total: 2 } - - diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/60_synonym_graph.yml b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/60_synonym_graph.yml index c4b1c85eddf60..e24dc918d449a 100644 --- a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/60_synonym_graph.yml +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/60_synonym_graph.yml @@ -1,6 +1,7 @@ setup: - do: indices.create: + include_type_name: false index: test body: settings: @@ -24,22 +25,19 @@ setup: tokenizer: standard filter: [ lowercase, graph_syns ] mappings: - test: - properties: - field: - type: text + properties: + field: + type: text - do: index: index: test - type: test id: 1 body: text: say wtf happened foo - do: index: index: test - type: test id: 2 body: text: bar baz what the fudge man @@ -47,7 +45,6 @@ setup: - do: index: index: test - type: test id: 3 body: text: wtf @@ -55,7 +52,6 @@ setup: - do: index: index: test - type: test id: 4 body: text: what is the name for fudge @@ -63,7 +59,6 @@ setup: - do: index: index: test - type: test id: 5 body: text: bar two three @@ -71,7 +66,6 @@ setup: - do: index: index: test - type: test id: 6 body: text: bar baz two three @@ -184,7 +178,6 @@ setup: - do: index: index: test - type: test id: 7 body: text: "WTFD!" @@ -192,7 +185,6 @@ setup: - do: index: index: test - type: test id: 8 body: text: "Weird Al's WHAT THE FUDGESICLE" diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.suggest/20_phrase.yml b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.suggest/20_phrase.yml index 9ac7ea7901dac..403d7dd2830fb 100644 --- a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.suggest/20_phrase.yml +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.suggest/20_phrase.yml @@ -3,6 +3,7 @@ setup: - do: indices.create: + include_type_name: false index: test body: settings: @@ -34,26 +35,24 @@ setup: min_shingle_size: 2 max_shingle_size: 2 mappings: - doc: - properties: - body: - type: text - analyzer: body - fields: - bigram: - type: text - analyzer: bigram - ngram: - type: text - analyzer: ngram - reverse: - type: text - analyzer: reverse + properties: + body: + type: text + analyzer: body + fields: + bigram: + type: text + analyzer: bigram + ngram: + type: text + analyzer: ngram + reverse: + type: text + analyzer: reverse - do: bulk: index: test - type: doc refresh: true body: | { "index": {} } @@ -226,4 +225,3 @@ setup: post_filter: reverse - match: {suggest.test.0.options.0.text: arthur king of the britons} - diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.suggest/30_synonyms.yml b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.suggest/30_synonyms.yml index cfac6742fbfb2..eb0bfc84a9223 100644 --- a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.suggest/30_synonyms.yml +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.suggest/30_synonyms.yml @@ -2,6 +2,7 @@ "suggestions with synonyms": - do: indices.create: + include_type_name: false index: test body: settings: @@ -16,16 +17,14 @@ type: synonym synonyms: [ "foo,renamed"] mappings: - test: - properties: - field: - type: completion - analyzer: suggest_analyzer_synonyms + properties: + field: + type: completion + analyzer: suggest_analyzer_synonyms - do: index: index: test - type: test id: 1 body: field: diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/termvectors/10_payloads.yml b/modules/analysis-common/src/test/resources/rest-api-spec/test/termvectors/10_payloads.yml index 489b55a63cd6f..0e9c5588371a8 100644 --- a/modules/analysis-common/src/test/resources/rest-api-spec/test/termvectors/10_payloads.yml +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/termvectors/10_payloads.yml @@ -3,6 +3,7 @@ # because there are no token filters that support payloads in core. - do: indices.create: + include_type_name: false index: test body: settings: @@ -14,17 +15,15 @@ tokenizer: standard filter: [type_as_payload] mappings: - doc: - properties: - text: - type: text - term_vector: with_positions_offsets_payloads - analyzer: has_payloads + properties: + text: + type: text + term_vector: with_positions_offsets_payloads + analyzer: has_payloads - do: index: index: test - type: doc id: 1 refresh: true body: @@ -33,7 +32,6 @@ - do: termvectors: index: test - type: doc id: 1 payloads: true - match: {term_vectors.text.field_statistics.sum_doc_freq: 5} diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/100_date_index_name_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/100_date_index_name_processor.yml index ccf83cc96bf7b..80598adf5f567 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/100_date_index_name_processor.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/100_date_index_name_processor.yml @@ -27,7 +27,6 @@ teardown: - do: index: index: events - type: event id: 1 pipeline: "1" body: { diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/110_sort.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/110_sort.yml index e18a0dcd25295..3c24d93ad8e58 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/110_sort.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/110_sort.yml @@ -26,7 +26,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: > @@ -37,6 +36,5 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.values: ["bar", "baz", "foo"] } diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/120_grok.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/120_grok.yml index 3d569d5c2c9b7..3551650b109f4 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/120_grok.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/120_grok.yml @@ -27,7 +27,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {field1: "123.42 400 "} @@ -35,7 +34,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.val: 123.42 } - match: { _source.status: 400 } @@ -66,7 +64,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {field1: ""} @@ -74,7 +71,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.msg: "foo" } @@ -103,7 +99,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {field1: ""} @@ -111,7 +106,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.msg: "foo" } diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/130_escape_dot.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/130_escape_dot.yml index 1d537ffa6b712..5fb416228f2c1 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/130_escape_dot.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/130_escape_dot.yml @@ -25,7 +25,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "1" body: { @@ -35,6 +34,5 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.foo.bar: "baz" } diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/140_json.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/140_json.yml index 81761ba509e10..95de56b2be3d4 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/140_json.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/140_json.yml @@ -50,7 +50,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "1" body: { @@ -65,7 +64,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.foo_object.hello: "world" } - match: { _source.foo_array.0: 1 } diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/150_kv.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/150_kv.yml index a1ecf10278c61..836243652b2e0 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/150_kv.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/150_kv.yml @@ -27,7 +27,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "1" body: { @@ -37,7 +36,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.goodbye: "everybody" } - match: { _source.hello: "world" } diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/160_urldecode.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/160_urldecode.yml index ae0439210b6a0..dc428d989a76f 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/160_urldecode.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/160_urldecode.yml @@ -25,7 +25,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "1" body: { @@ -35,6 +34,5 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.my_url: "https://elastic.co/" } diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/170_version.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/170_version.yml index 10c80c8e30525..b57cbbe3b7fb3 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/170_version.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/170_version.yml @@ -61,7 +61,6 @@ teardown: catch: conflict index: index: test - type: test id: 1 pipeline: "my_pipeline1" body: {} @@ -69,7 +68,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline2" body: {} diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/180_bytes_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/180_bytes_processor.yml index bc48720966c5f..1deeaa1edf7e3 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/180_bytes_processor.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/180_bytes_processor.yml @@ -27,7 +27,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {bytes_source_field: "1kb"} @@ -35,8 +34,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.bytes_source_field: "1kb" } - match: { _source.bytes_target_field: 1024 } - diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/190_script_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/190_script_processor.yml index bd55b764a95a5..3230fb37b43f7 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/190_script_processor.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/190_script_processor.yml @@ -27,7 +27,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {source_field: "1kb"} @@ -35,7 +34,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.source_field: "1kb" } - match: { _source.target_field: 1024 } @@ -62,7 +60,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {source_field: "FooBar"} @@ -70,7 +67,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.source_field: "FooBar" } - match: { _source.target_field: "foobar" } @@ -97,7 +93,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {source_field: "FooBar"} @@ -105,7 +100,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.source_field: "FooBar" } - match: { _source.target_field: "FOOBAR" } @@ -132,7 +126,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {source_field: "{\"foo\":\"bar\"}"} @@ -140,7 +133,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.source_field: "{\"foo\":\"bar\"}" } - match: { _source.target_field.foo: "bar" } @@ -167,7 +159,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {source_field: "{\"foo\":\"bar\"}"} @@ -175,7 +166,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.source_field: "{\"foo\":\"bar\"}" } - match: { _source.foo: "bar" } @@ -202,7 +192,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {source_field: "foo%20bar"} @@ -210,7 +199,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.source_field: "foo%20bar" } - match: { _source.target_field: "foo bar" } diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/200_default_pipeline.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/200_default_pipeline.yml index d4b39c5e99ac2..86f4821ddaa23 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/200_default_pipeline.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/200_default_pipeline.yml @@ -37,14 +37,12 @@ teardown: - do: index: index: test - type: test id: 1 body: {bytes_source_field: "1kb"} - do: get: index: test - type: test id: 1 - match: { _source.bytes_source_field: "1kb" } - match: { _source.bytes_target_field: 1024 } @@ -52,14 +50,12 @@ teardown: - do: index: index: test_alias - type: test id: 2 body: {bytes_source_field: "1kb"} - do: get: index: test - type: test id: 2 - match: { _source.bytes_source_field: "1kb" } - match: { _source.bytes_target_field: 1024 } @@ -67,7 +63,6 @@ teardown: - do: update: index: test - type: test id: 3 body: script: @@ -77,7 +72,6 @@ teardown: - do: get: index: test - type: test id: 3 - match: { _source.bytes_source_field: "1kb" } - match: { _source.bytes_target_field: 1024 } @@ -85,7 +79,6 @@ teardown: - do: update: index: test - type: test id: 4 body: script: @@ -96,7 +89,6 @@ teardown: - do: get: index: test - type: test id: 4 - match: { _source.bytes_source_field: "1kb" } - match: { _source.bytes_target_field: 1024 } @@ -104,7 +96,6 @@ teardown: - do: update: index: test - type: test id: 5 body: doc: { "bytes_source_field":"1kb" } @@ -112,7 +103,6 @@ teardown: - do: get: index: test - type: test id: 5 - match: { _source.bytes_source_field: "1kb" } - match: { _source.bytes_target_field: 1024 } @@ -123,20 +113,20 @@ teardown: bulk: refresh: true body: | - {"update":{"_id":"6","_index":"test","_type":"test"}} + {"update":{"_id":"6","_index":"test"}} {"script":"ctx._source.ran_script = true","upsert":{"bytes_source_field":"1kb"}} - {"update":{"_id":"7","_index":"test","_type":"test"}} + {"update":{"_id":"7","_index":"test"}} {"doc":{"bytes_source_field":"2kb"}, "doc_as_upsert":true} - {"update":{"_id":"8","_index":"test","_type":"test"}} + {"update":{"_id":"8","_index":"test"}} {"script": "ctx._source.ran_script = true","upsert":{"bytes_source_field":"3kb"}, "scripted_upsert" : true} - do: mget: body: docs: - - { _index: "test", _type: "_doc", _id: "6" } - - { _index: "test", _type: "_doc", _id: "7" } - - { _index: "test", _type: "_doc", _id: "8" } + - { _index: "test", _id: "6" } + - { _index: "test", _id: "7" } + - { _index: "test", _id: "8" } - match: { docs.0._index: "test" } - match: { docs.0._id: "6" } - match: { docs.0._source.bytes_source_field: "1kb" } @@ -156,15 +146,13 @@ teardown: - do: index: index: test - type: test id: 9 pipeline: "_none" body: {bytes_source_field: "1kb"} - + - do: get: index: test - type: test id: 9 - match: { _source.bytes_source_field: "1kb" } - is_false: _source.bytes_target_field @@ -173,7 +161,6 @@ teardown: catch: bad_request index: index: test - type: test id: 10 pipeline: "" body: {bytes_source_field: "1kb"} diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/200_dissect_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/200_dissect_processor.yml index 1a7c2e593d43c..709d4b9e62d43 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/200_dissect_processor.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/200_dissect_processor.yml @@ -27,7 +27,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {message: "foo bar baz"} @@ -35,7 +34,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.message: "foo bar baz" } - match: { _source.a: "foo" } @@ -87,4 +85,3 @@ teardown: } ] } - diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/210_conditional_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/210_conditional_processor.yml index 532519c4ca073..7b0999e4e2980 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/210_conditional_processor.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/210_conditional_processor.yml @@ -28,7 +28,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {bytes_source_field: "1kb", conditional_field: "bar"} @@ -36,7 +35,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.bytes_source_field: "1kb" } - match: { _source.conditional_field: "bar" } @@ -65,7 +63,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {bytes_source_field: "1kb", conditional_field: "bar"} @@ -73,9 +70,7 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.bytes_source_field: "1kb" } - match: { _source.conditional_field: "bar" } - is_false: _source.bytes_target_field - diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/210_pipeline_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/210_pipeline_processor.yml index 4efaaeb4091fa..d4ea46e26d3fa 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/210_pipeline_processor.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/210_pipeline_processor.yml @@ -54,7 +54,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "outer" body: {} @@ -62,7 +61,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.foo: "bar" } - match: { _source.baz: "blub" } @@ -105,7 +103,6 @@ teardown: catch: /illegal_state_exception/ index: index: test - type: test id: 1 pipeline: "outer" body: {} diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/220_drop_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/220_drop_processor.yml index accc30faa21e7..d1bb3b063a7c4 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/220_drop_processor.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/220_drop_processor.yml @@ -26,7 +26,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: { @@ -36,7 +35,6 @@ teardown: - do: index: index: test - type: test id: 2 pipeline: "my_pipeline" body: { @@ -47,14 +45,12 @@ teardown: catch: missing get: index: test - type: test id: 1 - match: { found: false } - do: get: index: test - type: test id: 2 - match: { _source.foo: "blub" } @@ -84,7 +80,6 @@ teardown: - do: index: index: test - type: test id: 3 pipeline: "my_pipeline_with_failure" body: { @@ -95,6 +90,5 @@ teardown: catch: missing get: index: test - type: test id: 3 - match: { found: false } diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml index 93dfa52196f35..b2e83c640388a 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml @@ -29,7 +29,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {date_source_field: "12/06/2010"} @@ -37,8 +36,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.date_source_field: "12/06/2010" } - match: { _source.date_target_field: "2010-06-12T00:00:00.000+02:00" } - diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/40_mutate.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/40_mutate.yml index 8150891ebd0db..11b6a64cd3fdf 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/40_mutate.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/40_mutate.yml @@ -84,7 +84,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: > @@ -103,7 +102,6 @@ teardown: - do: get: index: test - type: test id: 1 - is_false: _source.field_to_rename - is_false: _source.field_to_remove @@ -143,7 +141,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {field: "value"} @@ -151,8 +148,6 @@ teardown: - do: get: index: surprise - type: test id: 1 - length: { _source: 1 } - match: { _source.field: "value" } - diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/50_on_failure.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/50_on_failure.yml index 718b91ac1c111..4d74acdcab39c 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/50_on_failure.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/50_on_failure.yml @@ -47,7 +47,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {field1: "value1"} @@ -55,7 +54,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.field1: "value1" } - match: { _source._executed: true } @@ -105,7 +103,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {field1: "value1"} @@ -113,7 +110,6 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.field1: "value1" } - match: { _source.foofield: "exists" } @@ -202,7 +198,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {} @@ -210,6 +205,5 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.field: "value" } diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/60_fail.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/60_fail.yml index e080991e13c12..6b580a09239ec 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/60_fail.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/60_fail.yml @@ -27,7 +27,6 @@ teardown: catch: request index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {} @@ -61,7 +60,6 @@ teardown: - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {} @@ -69,7 +67,5 @@ teardown: - do: get: index: test - type: test id: 1 - match: { _source.error_message: "fail_processor_ran" } - diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/70_bulk.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/70_bulk.yml index cf428b3252440..5b442456e64d0 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/70_bulk.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/70_bulk.yml @@ -50,13 +50,11 @@ teardown: body: - index: _index: test_index - _type: test_type _id: test_id1 pipeline: pipeline1 - f1: v1 - index: _index: test_index - _type: test_type _id: test_id2 - f1: v2 - gte: { ingest_took: 0 } @@ -64,7 +62,6 @@ teardown: - do: get: index: test_index - type: test_type id: test_id1 - match: {_source.field1: value1} @@ -73,7 +70,6 @@ teardown: - do: get: index: test_index - type: test_type id: test_id2 - is_false: _source.field1 @@ -106,12 +102,10 @@ teardown: body: - index: _index: test_index - _type: test_type _id: test_id1 - f1: v1 - index: _index: test_index - _type: test_type _id: test_id2 pipeline: pipeline2 - f1: v2 @@ -138,7 +132,6 @@ teardown: - do: get: index: test_index - type: test_type id: test_id1 - match: {_source.field1: value1} @@ -147,9 +140,7 @@ teardown: - do: get: index: test_index - type: test_type id: test_id2 - is_false: _source.field1 - match: {_source.field2: value2} - diff --git a/modules/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml b/modules/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml index d5c1f8e5c489f..78368b8703a6d 100644 --- a/modules/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml +++ b/modules/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml @@ -19,7 +19,6 @@ - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {field1: "128.101.101.101"} @@ -27,7 +26,6 @@ - do: get: index: test - type: test id: 1 - match: { _source.field1: "128.101.101.101" } - length: { _source.geoip: 6 } @@ -65,7 +63,6 @@ - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {field1: "128.101.101.101"} @@ -73,7 +70,6 @@ - do: get: index: test - type: test id: 1 - match: { _source.field1: "128.101.101.101" } - length: { _source.geoip: 9 } @@ -110,7 +106,6 @@ - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {field1: "128.101.101.101"} @@ -118,7 +113,6 @@ - do: get: index: test - type: test id: 1 - match: { _source.field1: "128.101.101.101" } - length: { _source.geoip: 2 } @@ -163,7 +157,6 @@ - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: { field1: "80.231.5.0" } @@ -171,7 +164,6 @@ - do: get: index: test - type: test id: 1 - match: { _source.field1: "80.231.5.0" } - is_false: _source.geoip @@ -179,7 +171,6 @@ - do: index: index: test - type: test id: 2 pipeline: "my_pipeline" body: { field1: "128.101.101.101" } @@ -187,7 +178,6 @@ - do: get: index: test - type: test id: 2 - match: { _source.field1: "128.101.101.101" } - length: { _source.geoip: 6 } @@ -221,7 +211,6 @@ - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {field1: "82.171.64.0"} @@ -229,7 +218,6 @@ - do: get: index: test - type: test id: 1 - match: { _source.field1: "82.171.64.0" } - length: { _source.geoip: 3 } diff --git a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml b/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml index 0964e69a99b6d..28c218edd6935 100644 --- a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml +++ b/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml @@ -19,7 +19,6 @@ - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36"} @@ -27,7 +26,6 @@ - do: get: index: test - type: test id: 1 - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } - match: { _source.user_agent.name: "Chrome" } @@ -63,7 +61,6 @@ - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36"} @@ -71,7 +68,6 @@ - do: get: index: test - type: test id: 1 - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } - match: { _source.field2.os: "Mac OS X 10.9.2" } diff --git a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml b/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml index 5a6cd8f7a86f6..22df584e13166 100644 --- a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml +++ b/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml @@ -20,7 +20,6 @@ - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: {field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36"} @@ -28,7 +27,6 @@ - do: get: index: test - type: test id: 1 - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } - match: { _source.user_agent.name: "Test" } diff --git a/modules/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/20_search.yml b/modules/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/20_search.yml index a8850263d40ff..197daf5357be0 100644 --- a/modules/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/20_search.yml +++ b/modules/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/20_search.yml @@ -2,17 +2,16 @@ setup: - do: indices.create: + include_type_name: false index: test123 body: mappings: - test: - properties: - age: - type: long + properties: + age: + type: long - do: index: index: test123 - type: test id: 1 body: { age: 23 } @@ -33,4 +32,3 @@ setup: source: 'doc["age"].value + 19' - match: { hits.hits.0.fields.my_field.0: 42.0 } - diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/20_render_search_template.yml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/20_render_search_template.yml index 78676787dda88..946b63a65d923 100644 --- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/20_render_search_template.yml +++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/20_render_search_template.yml @@ -109,13 +109,11 @@ - do: index: index: test - type: testtype id: 1 body: { "text": "value1_foo" } - do: index: index: test - type: testtype id: 2 body: { "text": "value2_foo value3_foo" } - do: diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/25_custom_functions.yml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/25_custom_functions.yml index f58d40402b147..57177d6eb351f 100644 --- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/25_custom_functions.yml +++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/25_custom_functions.yml @@ -8,17 +8,16 @@ "source": { "query": { "match": { - "url": "https://localhost:9200/{{#url}}{{index}}{{/url}}/{{#url}}{{type}}{{/url}}/_search" + "url": "https://localhost:9200/{{#url}}{{index}}{{/url}}/_search" } } }, "params": { - "index": "", - "type" : "métriques" + "index": "" } } - - match: { template_output.query.match.url: "https://localhost:9200/%3Clogstash-%7Bnow%2Fd-2d%7D%3E/m%C3%A9triques/_search" } + - match: { template_output.query.match.url: "https://localhost:9200/%3Clogstash-%7Bnow%2Fd-2d%7D%3E/_search" } --- "Rendering using {{url}} and {{join}} functions": diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml index de66440cc2189..bc039a5ba7c50 100644 --- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml +++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml @@ -4,13 +4,11 @@ - do: index: index: test - type: testtype id: 1 body: { "text": "value1" } - do: index: index: test - type: testtype id: 2 body: { "text": "value2" } - do: @@ -57,28 +55,24 @@ - do: index: index: test - type: type id: 1 body: { "theField": "foo" } - do: index: index: test - type: type id: 2 body: { "theField": "foo 2" } - do: index: index: test - type: type id: 3 body: { "theField": "foo 3" } - do: index: index: test - type: type id: 4 body: { "theField": "foo 4" } @@ -87,7 +81,6 @@ - do: index: index: otherindex - type: type id: 5 body: { "otherField": "foo" } - do: @@ -142,7 +135,6 @@ - do: index: index: test - type: type id: 1 body: {} @@ -159,4 +151,3 @@ - match: { hits.total.value: 0 } - match: { hits.total.relation: eq } - diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml index 077760d03e8fa..a5072d529b9b5 100644 --- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml +++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml @@ -4,28 +4,24 @@ setup: - do: index: index: index_1 - type: test id: 1 body: { foo: bar } - do: index: index: index_1 - type: test id: 2 body: { foo: baz } - do: index: index: index_1 - type: test id: 3 body: { foo: foo } - do: index: index: index_2 - type: test id: 1 body: { foo: foo } @@ -205,4 +201,3 @@ setup: - match: { responses.1.hits.total.relation: eq } - match: { responses.2.hits.total.value: 1 } - match: { responses.1.hits.total.relation: eq } - diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/15_update.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/15_update.yml index f2e1cb616b980..15fcf54ce5a2a 100644 --- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/15_update.yml +++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/15_update.yml @@ -4,7 +4,6 @@ - do: index: index: test_1 - type: test id: 1 body: foo: bar @@ -13,7 +12,6 @@ - do: update: index: test_1 - type: test id: 1 body: script: @@ -22,14 +20,13 @@ params: { bar: 'xxx' } - match: { _index: test_1 } - - match: { _type: test } + - match: { _type: _doc } - match: { _id: "1" } - match: { _version: 2 } - do: get: index: test_1 - type: test id: 1 - match: { _source.foo: xxx } @@ -38,7 +35,6 @@ - do: update: index: test_1 - type: test id: 1 body: script: @@ -46,14 +42,13 @@ source: "ctx._source.foo = 'yyy'" - match: { _index: test_1 } - - match: { _type: test } + - match: { _type: _doc } - match: { _id: "1" } - match: { _version: 3 } - do: get: index: test_1 - type: test id: 1 - match: { _source.foo: yyy } @@ -62,7 +57,6 @@ - do: update: index: test_1 - type: test id: 1 body: script: @@ -70,14 +64,13 @@ source: "ctx._source.missing_length = ctx._source.missing?.length()" - match: { _index: test_1 } - - match: { _type: test } + - match: { _type: _doc } - match: { _id: "1" } - match: { _version: 4 } - do: get: index: test_1 - type: test id: 1 - match: { _source.foo: yyy } @@ -88,7 +81,6 @@ - do: update: index: test_1 - type: test id: 1 body: script: @@ -96,14 +88,13 @@ source: "ctx._source.foo_length = ctx._source.foo?.length()" - match: { _index: test_1 } - - match: { _type: test } + - match: { _type: _doc } - match: { _id: "1" } - match: { _version: 5 } - do: get: index: test_1 - type: test id: 1 - match: { _source.foo: yyy } @@ -117,7 +108,6 @@ - do: index: index: test_1 - type: test id: 2 body: foo: bar @@ -127,7 +117,6 @@ catch: bad_request update: index: test_1 - type: test id: 2 body: script: diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/20_scriptfield.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/20_scriptfield.yml index 416d4457f317b..470ace98dd329 100644 --- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/20_scriptfield.yml +++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/20_scriptfield.yml @@ -3,25 +3,24 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - test: - properties: - foo: - type: keyword - missing: - type: keyword - date: - type: date - format: yyyy/MM/dd - dates: - type: date - format: yyyy/MM/dd + properties: + foo: + type: keyword + missing: + type: keyword + date: + type: date + format: yyyy/MM/dd + dates: + type: date + format: yyyy/MM/dd - do: index: index: test - type: test id: 1 body: { "foo": "aaa", @@ -156,4 +155,3 @@ setup: - match: { error.failed_shards.0.reason.caused_by.reason: "While loop has no escape." } - match: { error.failed_shards.0.reason.type: "script_exception" } - match: { error.failed_shards.0.reason.reason: "compile error" } - diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/25_script_upsert.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/25_script_upsert.yml index 9622e6f4d3168..6023c7df64b09 100644 --- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/25_script_upsert.yml +++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/25_script_upsert.yml @@ -4,7 +4,6 @@ - do: update: index: test_1 - type: test id: 1 body: script: @@ -16,7 +15,6 @@ - do: get: index: test_1 - type: test id: 1 - match: { _source.foo: baz } @@ -25,7 +23,6 @@ - do: update: index: test_1 - type: test id: 1 body: script: @@ -37,7 +34,6 @@ - do: get: index: test_1 - type: test id: 1 - match: { _source.foo: xxx } @@ -45,7 +41,6 @@ - do: update: index: test_1 - type: test id: 2 body: script: @@ -58,7 +53,6 @@ - do: get: index: test_1 - type: test id: 2 - match: { _source.foo: xxx } @@ -66,7 +60,6 @@ - do: update: index: test_1 - type: test id: 3 body: script: @@ -78,9 +71,6 @@ - do: get: index: test_1 - type: test id: 3 - match: { _source.has_now: true } - - diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/30_search.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/30_search.yml index 367a63d31339d..0ce1e369cb7c5 100644 --- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/30_search.yml +++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/30_search.yml @@ -4,19 +4,16 @@ - do: index: index: test - type: test id: 1 body: { "test": "value beck", "num1": 1.0, "bool": true } - do: index: index: test - type: test id: 2 body: { "test": "value beck", "num1": 2.0, "bool": false } - do: index: index: test - type: test id: 3 body: { "test": "value beck", "num1": 3.0, "bool": true } - do: @@ -34,7 +31,7 @@ lang: painless script_fields: sNum1: - script: + script: source: "doc['num1'].value;" lang: painless sort: @@ -86,7 +83,7 @@ script_fields: sNum1: - script: + script: source: "doc['num1'].value;" lang: painless sort: @@ -118,13 +115,11 @@ - do: index: index: test - type: test id: 1 body: { "test": "value beck", "num1": 1.0 } - do: index: index: test - type: test id: 2 body: { "test": "value beck", "num1": 2.0 } - do: @@ -277,7 +272,6 @@ - do: index: index: test - type: test id: 1 body: { "dummy_field": 1 } - do: @@ -328,7 +322,6 @@ - do: index: index: test - type: test id: 1 body: { "dummy_field": 1 } - do: @@ -368,7 +361,6 @@ - do: index: index: test - type: test id: 1 body: { "f": 42 } - do: @@ -382,7 +374,7 @@ body: script_fields: foobar: - script: + script: source: "doc['f'].size()" lang: painless @@ -396,7 +388,6 @@ - do: index: index: test - type: test id: 1 body: { "dummy_field": 1 } - do: @@ -433,7 +424,6 @@ - do: index: index: test - type: test id: 1 body: { "genre": 1 } @@ -469,7 +459,6 @@ - do: index: index: test - type: test id: 1 body: { "test": "value beck", "num1": 1.0 } - do: diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/40_disabled.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/40_disabled.yml index bcf02f657b0aa..245f14641f7a5 100644 --- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/40_disabled.yml +++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/40_disabled.yml @@ -4,7 +4,6 @@ - do: index: index: test_1 - type: test id: 1 body: foo: bar @@ -14,7 +13,6 @@ catch: /Regexes are disabled. Set \[script.painless.regex.enabled\] to \[true\] in elasticsearch.yaml to allow them. Be careful though, regexes break out of Painless's protection against deep recursion and long loops./ update: index: test_1 - type: test id: 1 body: script: diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yml index 01e3ca907bf02..b112408396e53 100644 --- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yml +++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yml @@ -1,48 +1,47 @@ setup: - do: indices.create: + include_type_name: false index: test body: settings: number_of_shards: 1 mappings: - test: - properties: - boolean: - type: boolean - date: - type: date - geo_point: - type: geo_point - ip: - type: ip - keyword: - type: keyword - long: - type: long - integer: - type: integer - short: - type: short - byte: - type: byte - double: - type: double - float: - type: float - half_float: - type: half_float - scaled_float: - type: scaled_float - scaling_factor: 100 - token_count: - type: token_count - analyzer: standard + properties: + boolean: + type: boolean + date: + type: date + geo_point: + type: geo_point + ip: + type: ip + keyword: + type: keyword + long: + type: long + integer: + type: integer + short: + type: short + byte: + type: byte + double: + type: double + float: + type: float + half_float: + type: half_float + scaled_float: + type: scaled_float + scaling_factor: 100 + token_count: + type: token_count + analyzer: standard - do: index: index: test - type: test id: 1 body: boolean: true diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/60_script_doc_values_binary.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/60_script_doc_values_binary.yml index 04ca0307776c4..f8ad782665e98 100644 --- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/60_script_doc_values_binary.yml +++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/60_script_doc_values_binary.yml @@ -4,14 +4,14 @@ features: ["headers"] - do: indices.create: + include_type_name: false index: test body: mappings: - test: - properties: - binary: - type: binary - doc_values: true + properties: + binary: + type: binary + doc_values: true - do: #set the header so we won't randomize it @@ -19,7 +19,6 @@ Content-Type: application/json index: index: test - type: test id: 1 body: binary: U29tZSBiaW5hcnkgYmxvYg== @@ -46,4 +45,3 @@ script: source: "doc['binary'].value.utf8ToString()" - match: { hits.hits.0.fields.field.0: "Some binary blob" } - diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/70_execute_painless_scripts.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/70_execute_painless_scripts.yml index 1e34a776189b8..d06cf45dadd3c 100644 --- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/70_execute_painless_scripts.yml +++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/70_execute_painless_scripts.yml @@ -1,17 +1,17 @@ setup: - do: indices.create: + include_type_name: false index: my-index body: mappings: - doc: - properties: - rank: - type: long - field: - type: keyword - text: - type: text + properties: + rank: + type: long + field: + type: keyword + text: + type: text --- "Execute with defaults": diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/70_mov_fn_agg.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/70_mov_fn_agg.yml index 630c2fcd777c6..16432f9e70e7a 100644 --- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/70_mov_fn_agg.yml +++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/70_mov_fn_agg.yml @@ -6,15 +6,15 @@ setup: reason: "moving_fn added in 6.4.0" - do: indices.create: + include_type_name: false index: test body: mappings: - _doc: - properties: - value_field: - type: integer - date: - type: date + properties: + value_field: + type: integer + date: + type: date - do: bulk: @@ -22,37 +22,31 @@ setup: body: - index: _index: test - _type: _doc _id: 1 - date: "2017-01-01T00:00:00" value_field: 1 - index: _index: test - _type: _doc _id: 2 - date: "2017-01-02T00:00:00" value_field: 2 - index: _index: test - _type: _doc _id: 3 - date: "2017-01-03T00:00:00" value_field: 3 - index: _index: test - _type: _doc _id: 4 - date: "2017-01-04T00:00:00" value_field: 4 - index: _index: test - _type: _doc _id: 5 - date: "2017-01-05T00:00:00" value_field: 5 - index: _index: test - _type: _doc _id: 6 - date: "2017-01-06T00:00:00" value_field: 6 @@ -316,9 +310,3 @@ setup: - match: { hits.total: 6 } - length: { hits.hits: 0 } - - - - - - diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/80_script_score.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/80_script_score.yml index fb6f40694b271..24dd8b87d8b8f 100644 --- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/80_script_score.yml +++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/80_script_score.yml @@ -80,31 +80,28 @@ setup: "Random functions": - do: indices.create: + include_type_name: false index: test body: settings: number_of_shards: 2 mappings: - _doc: - properties: - f1: - type: keyword + properties: + f1: + type: keyword - do: index: index: test - type: _doc id: 1 body: {"f1": "v1"} - do: index: index: test - type: _doc id: 2 body: {"f1": "v2"} - do: index: index: test - type: _doc id: 3 body: {"f1": "v3"} @@ -139,27 +136,25 @@ setup: "Decay geo functions": - do: indices.create: + include_type_name: false index: test body: settings: number_of_shards: 1 mappings: - _doc: - properties: - text-location: - type: keyword - location: - type: geo_point + properties: + text-location: + type: keyword + location: + type: geo_point - do: index: index: test - type: _doc id: 1 body: { "text-location": "location1", "location" : {"lat" : 40.24, "lon" : -70.24} } - do: index: index: test - type: _doc id: 2 body: { "text-location": "location2", "location" : {"lat" : 40.12, "lon" : -70.12} } - do: @@ -239,13 +234,11 @@ setup: - do: index: index: test - type: _doc id: 1 body: { "date": "2018-01-01T02:00:00Z"} - do: index: index: test - type: _doc id: 2 body: { "date": "2018-01-01T01:00:00Z" } - do: @@ -332,7 +325,6 @@ setup: - do: index: index: test - type: _doc id: 1 body: { "ival" : 40, "lval" : 40, "fval": 40.0, "dval": 40.0} @@ -340,7 +332,6 @@ setup: - do: index: index: test - type: _doc id: 2 body: { "ival" : [50, 40, 20], "lval" : [50, 40, 20], "fval" : [50.0, 40.0, 20.0], "dval" : [50.0, 40.0, 20.0] } - do: diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/90_interval_query_filter.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/90_interval_query_filter.yml index 8aa10e92a245b..270f5682aacb4 100644 --- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/90_interval_query_filter.yml +++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/90_interval_query_filter.yml @@ -5,25 +5,25 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - test: - properties: - text: - type: text - analyzer: standard + properties: + text: + type: text + analyzer: standard - do: bulk: refresh: true body: - - '{"index": {"_index": "test", "_type": "test", "_id": "1"}}' + - '{"index": {"_index": "test", "_id": "1"}}' - '{"text" : "Some like it hot, some like it cold"}' - - '{"index": {"_index": "test", "_type": "test", "_id": "2"}}' + - '{"index": {"_index": "test", "_id": "2"}}' - '{"text" : "Its cold outside, theres no kind of atmosphere"}' - - '{"index": {"_index": "test", "_type": "test", "_id": "3"}}' + - '{"index": {"_index": "test", "_id": "3"}}' - '{"text" : "Baby its cold there outside"}' - - '{"index": {"_index": "test", "_type": "test", "_id": "4"}}' + - '{"index": {"_index": "test", "_id": "4"}}' - '{"text" : "Outside it is cold and wet"}' --- @@ -42,5 +42,3 @@ setup: source: "interval.start > 3" - match: { hits.total.value: 1 } - - diff --git a/modules/mapper-extras/src/test/resources/rest-api-spec/test/dense-vector/10_indexing.yml b/modules/mapper-extras/src/test/resources/rest-api-spec/test/dense-vector/10_indexing.yml index ef31d0f45e240..fd554fb16dbbd 100644 --- a/modules/mapper-extras/src/test/resources/rest-api-spec/test/dense-vector/10_indexing.yml +++ b/modules/mapper-extras/src/test/resources/rest-api-spec/test/dense-vector/10_indexing.yml @@ -5,15 +5,15 @@ setup: - do: indices.create: + include_type_name: false index: test-index body: settings: number_of_replicas: 0 mappings: - _doc: - properties: - my_dense_vector: - type: dense_vector + properties: + my_dense_vector: + type: dense_vector --- @@ -21,7 +21,6 @@ setup: - do: index: index: test-index - type: _doc id: 1 body: my_dense_vector: [1.5, -10, 3455, 345452.4545] diff --git a/modules/mapper-extras/src/test/resources/rest-api-spec/test/rank_feature/10_basic.yml b/modules/mapper-extras/src/test/resources/rest-api-spec/test/rank_feature/10_basic.yml index f02e64b7f5cec..28cd9a4abc045 100644 --- a/modules/mapper-extras/src/test/resources/rest-api-spec/test/rank_feature/10_basic.yml +++ b/modules/mapper-extras/src/test/resources/rest-api-spec/test/rank_feature/10_basic.yml @@ -5,23 +5,22 @@ setup: - do: indices.create: + include_type_name: false index: test body: settings: number_of_replicas: 0 mappings: - _doc: - properties: - pagerank: - type: rank_feature - url_length: - type: rank_feature - positive_score_impact: false + properties: + pagerank: + type: rank_feature + url_length: + type: rank_feature + positive_score_impact: false - do: index: index: test - type: _doc id: 1 body: pagerank: 10 @@ -30,9 +29,8 @@ setup: - do: index: index: test - type: _doc id: 2 - body: + body: pagerank: 100 url_length: 20 diff --git a/modules/mapper-extras/src/test/resources/rest-api-spec/test/rank_features/10_basic.yml b/modules/mapper-extras/src/test/resources/rest-api-spec/test/rank_features/10_basic.yml index 66b585f6ed76d..1c1f021d0fd8e 100644 --- a/modules/mapper-extras/src/test/resources/rest-api-spec/test/rank_features/10_basic.yml +++ b/modules/mapper-extras/src/test/resources/rest-api-spec/test/rank_features/10_basic.yml @@ -5,20 +5,19 @@ setup: - do: indices.create: + include_type_name: false index: test body: settings: number_of_replicas: 0 mappings: - _doc: - properties: - tags: - type: rank_features + properties: + tags: + type: rank_features - do: index: index: test - type: _doc id: 1 body: tags: @@ -28,9 +27,8 @@ setup: - do: index: index: test - type: _doc id: 2 - body: + body: tags: bar: 6 quux: 10 diff --git a/modules/mapper-extras/src/test/resources/rest-api-spec/test/scaled_float/10_basic.yml b/modules/mapper-extras/src/test/resources/rest-api-spec/test/scaled_float/10_basic.yml index f40ee53a843c2..9badd5ca3018d 100644 --- a/modules/mapper-extras/src/test/resources/rest-api-spec/test/scaled_float/10_basic.yml +++ b/modules/mapper-extras/src/test/resources/rest-api-spec/test/scaled_float/10_basic.yml @@ -1,42 +1,38 @@ setup: - do: indices.create: + include_type_name: false index: test body: settings: number_of_replicas: 0 mappings: - doc: - "properties": - "number": - "type" : "scaled_float" - "scaling_factor": 100 + "properties": + "number": + "type" : "scaled_float" + "scaling_factor": 100 - do: index: index: test - type: doc id: 1 body: { "number" : 1 } - do: index: index: test - type: doc id: 2 body: { "number" : 1.53 } - do: index: index: test - type: doc id: 3 body: { "number" : -2.1 } - do: index: index: test - type: doc id: 4 body: { "number" : 1.53 } @@ -107,4 +103,3 @@ setup: - match: { hits.total: 4 } - match: { hits.hits.0._id: "3" } - diff --git a/modules/mapper-extras/src/test/resources/rest-api-spec/test/sparse-vector/10_indexing.yml b/modules/mapper-extras/src/test/resources/rest-api-spec/test/sparse-vector/10_indexing.yml index 87d599e9cb078..ef78a48fab5a1 100644 --- a/modules/mapper-extras/src/test/resources/rest-api-spec/test/sparse-vector/10_indexing.yml +++ b/modules/mapper-extras/src/test/resources/rest-api-spec/test/sparse-vector/10_indexing.yml @@ -5,15 +5,15 @@ setup: - do: indices.create: + include_type_name: false index: test-index body: settings: number_of_replicas: 0 mappings: - _doc: - properties: - my_sparse_vector: - type: sparse_vector + properties: + my_sparse_vector: + type: sparse_vector --- @@ -21,7 +21,6 @@ setup: - do: index: index: test-index - type: _doc id: 1 body: my_sparse_vector: { "50" : 1.8, "2" : -0.4, "10" : 1000.3, "4545" : -0.00004} diff --git a/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml b/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml index d120504f18c6d..7d3fb36bafc93 100644 --- a/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml +++ b/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml @@ -1,27 +1,25 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - doc: - properties: - join_field: - type: join - relations: - parent: child + properties: + join_field: + type: join + relations: + parent: child - do: index: index: test - type: doc id: 1 body: {"foo": "bar", "join_field": {"name" : "parent"} } - do: index: index: test - type: doc id: 2 routing: 1 body: {"bar": "baz", "join_field": { "name" : "child", "parent": "1"} } diff --git a/modules/parent-join/src/test/resources/rest-api-spec/test/20_parent_join.yml b/modules/parent-join/src/test/resources/rest-api-spec/test/20_parent_join.yml index 66f9c1f186935..27af9794843d9 100644 --- a/modules/parent-join/src/test/resources/rest-api-spec/test/20_parent_join.yml +++ b/modules/parent-join/src/test/resources/rest-api-spec/test/20_parent_join.yml @@ -1,31 +1,28 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - doc: - properties: - join_field: { "type": "join", "relations": { "parent": "child", "child": "grand_child" } } + properties: + join_field: { "type": "join", "relations": { "parent": "child", "child": "grand_child" } } - do: index: index: test - type: doc id: 1 body: { "join_field": { "name": "parent" } } - do: index: index: test - type: doc id: 2 body: { "join_field": { "name": "parent" } } - do: index: index: test - type: doc id: 3 routing: 1 body: { "join_field": { "name": "child", "parent": "1" } } @@ -33,7 +30,6 @@ setup: - do: index: index: test - type: doc id: 4 routing: 1 body: { "join_field": { "name": "child", "parent": "1" } } @@ -41,7 +37,6 @@ setup: - do: index: index: test - type: doc id: 5 routing: 1 body: { "join_field": { "name": "child", "parent": "2" } } @@ -49,7 +44,6 @@ setup: - do: index: index: test - type: doc id: 6 routing: 1 body: { "join_field": { "name": "grand_child", "parent": "5" } } @@ -66,35 +60,35 @@ setup: - match: { hits.total: 6 } - match: { hits.hits.0._index: "test" } - - match: { hits.hits.0._type: "doc" } + - match: { hits.hits.0._type: "_doc" } - match: { hits.hits.0._id: "3" } - match: { hits.hits.0._source.join_field.name: "child" } - match: { hits.hits.0._source.join_field.parent: "1" } - is_false: hits.hits.0.fields.join_field#child } - match: { hits.hits.1._index: "test" } - - match: { hits.hits.1._type: "doc" } + - match: { hits.hits.1._type: "_doc" } - match: { hits.hits.1._id: "4" } - match: { hits.hits.1._source.join_field.name: "child" } - match: { hits.hits.1._source.join_field.parent: "1" } - is_false: hits.hits.1.fields.join_field#child } - match: { hits.hits.2._index: "test" } - - match: { hits.hits.2._type: "doc" } + - match: { hits.hits.2._type: "_doc" } - match: { hits.hits.2._id: "5" } - match: { hits.hits.2._source.join_field.name: "child" } - match: { hits.hits.2._source.join_field.parent: "2" } - is_false: hits.hits.2.fields.join_field#child } - match: { hits.hits.3._index: "test" } - - match: { hits.hits.3._type: "doc" } + - match: { hits.hits.3._type: "_doc" } - match: { hits.hits.3._id: "6" } - match: { hits.hits.3._source.join_field.name: "grand_child" } - match: { hits.hits.3._source.join_field.parent: "5" } - match: { hits.hits.4._index: "test" } - - match: { hits.hits.4._type: "doc" } + - match: { hits.hits.4._type: "_doc" } - match: { hits.hits.4._id: "1" } - match: { hits.hits.4._source.join_field.name: "parent" } - is_false: hits.hits.4._source.join_field.parent - match: { hits.hits.5._index: "test" } - - match: { hits.hits.5._type: "doc" } + - match: { hits.hits.5._type: "_doc" } - match: { hits.hits.5._id: "2" } - match: { hits.hits.5._source.join_field.name: "parent" } - is_false: hits.hits.5._source.join_field.parent @@ -113,14 +107,12 @@ setup: - match: { hits.total: 2 } - match: { hits.hits.0._index: "test" } - - match: { hits.hits.0._type: "doc" } + - match: { hits.hits.0._type: "_doc" } - match: { hits.hits.0._id: "3" } - match: { hits.hits.0._source.join_field.name: "child" } - match: { hits.hits.0._source.join_field.parent: "1" } - match: { hits.hits.1._index: "test" } - - match: { hits.hits.1._type: "doc" } + - match: { hits.hits.1._type: "_doc" } - match: { hits.hits.1._id: "4" } - match: { hits.hits.1._source.join_field.name: "child" } - match: { hits.hits.1._source.join_field.parent: "1" } - - diff --git a/modules/percolator/src/test/resources/rest-api-spec/test/10_basic.yml b/modules/percolator/src/test/resources/rest-api-spec/test/10_basic.yml index 9dbe3dfe0d1f4..8c39ed8278a69 100644 --- a/modules/percolator/src/test/resources/rest-api-spec/test/10_basic.yml +++ b/modules/percolator/src/test/resources/rest-api-spec/test/10_basic.yml @@ -2,20 +2,19 @@ "Test percolator basics via rest": - do: indices.create: + include_type_name: false index: queries_index body: mappings: - doc: - properties: - query: - type: percolator - foo: - type: keyword + properties: + query: + type: percolator + foo: + type: keyword - do: index: index: queries_index - type: doc id: test_percolator body: query: diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml index ebe23ae53f411..b9001cb782a80 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml @@ -1,4 +1,4 @@ -setup: +setup: - do: indices.create: index: foo @@ -9,28 +9,24 @@ setup: - do: index: index: foo - type: bar id: doc1 body: { "text": "berlin" } - do: index: index: foo - type: bar id: doc2 body: { "text": "amsterdam" } - do: index: index: foo - type: bar id: doc3 body: { "text": "amsterdam" } - + - do: index: index: foo - type: bar id: doc4 body: { "text": "something about amsterdam and berlin" } @@ -52,7 +48,7 @@ setup: - do: rank_eval: index: foo, - body: { + body: { "requests" : [ { "id": "amsterdam_query", @@ -75,7 +71,7 @@ setup: - match: { details.amsterdam_query.metric_score: 1.0} - match: { details.amsterdam_query.unrated_docs: [ {"_index": "foo", "_id": "doc4"}]} - match: { details.amsterdam_query.metric_details.precision: {"relevant_docs_retrieved": 2, "docs_retrieved": 2}} - + - length: { details.amsterdam_query.hits: 3} - match: { details.amsterdam_query.hits.0.hit._id: "doc2"} - match: { details.amsterdam_query.hits.0.rating: 1} @@ -83,7 +79,7 @@ setup: - match: { details.amsterdam_query.hits.1.rating: 1} - match: { details.amsterdam_query.hits.2.hit._id: "doc4"} - is_false: details.amsterdam_query.hits.2.rating - + - match: { details.berlin_query.metric_score: 1.0} - match: { details.berlin_query.unrated_docs: [ {"_index": "foo", "_id": "doc4"}]} - match: { details.berlin_query.metric_details.precision: {"relevant_docs_retrieved": 1, "docs_retrieved": 1}} @@ -99,7 +95,7 @@ setup: - do: rank_eval: index: alias - body: { + body: { "requests" : [ { "id": "amsterdam_query", @@ -142,7 +138,7 @@ setup: { "id" : "berlin_query", "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 }, - # doc1 should be returned in first position, doc3 in second, so reciprocal rank is 1/2 + # doc1 should be returned in first position, doc3 in second, so reciprocal rank is 1/2 "ratings": [{"_index": "foo", "_id": "doc4", "rating": 1}] } ], @@ -183,7 +179,7 @@ setup: "ratings": [{"_index": "foo", "_id": "doc4", "rating": 1}] } ], - "metric" : { + "metric" : { "expected_reciprocal_rank": { "maximum_relevance" : 1, "k" : 5 diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yml index 1b159775d5c94..90094baabb9db 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yml @@ -8,42 +8,36 @@ - do: index: index: foo - type: bar id: doc1 body: { "bar": 1 } - do: index: index: foo - type: bar id: doc2 body: { "bar": 2 } - do: index: index: foo - type: bar id: doc3 body: { "bar": 3 } - do: index: index: foo - type: bar id: doc4 body: { "bar": 4 } - do: index: index: foo - type: bar id: doc5 body: { "bar": 5 } - do: index: index: foo - type: bar id: doc6 body: { "bar": 6 } @@ -91,7 +85,7 @@ {"_index" : "foo", "_id" : "doc4", "rating": 0}, {"_index" : "foo", "_id" : "doc5", "rating": 1}, {"_index" : "foo", "_id" : "doc6", "rating": 2}] - }, + }, ], "metric" : { "dcg": { }} } @@ -103,7 +97,7 @@ - match: {details.dcg_query_reverse.unrated_docs: [ ]} # if we mix both, we should get the average - + - do: rank_eval: body: { @@ -129,7 +123,7 @@ {"_index" : "foo", "_id" : "doc4", "rating": 0}, {"_index" : "foo", "_id" : "doc5", "rating": 1}, {"_index" : "foo", "_id" : "doc6", "rating": 2}] - }, + }, ], "metric" : { "dcg": { }} } diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml index 42627a2590e1c..b9f55ed12ad7e 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml @@ -8,7 +8,6 @@ - do: index: index: foo - type: bar id: doc1 body: { "bar": 1 } diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/40_rank_eval_templated.yml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/40_rank_eval_templated.yml index fef25c3fc41a5..57d5aa5642ef6 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/40_rank_eval_templated.yml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/40_rank_eval_templated.yml @@ -10,42 +10,36 @@ setup: - do: index: index: test - type: _doc id: 1 body: { "text": "berlin", "title" : "Berlin, Germany" } - do: index: index: test - type: _doc id: 2 body: { "text": "amsterdam" } - do: index: index: test - type: _doc id: 3 body: { "text": "amsterdam" } - do: index: index: test - type: _doc id: 4 body: { "text": "amsterdam" } - do: index: index: test - type: _doc id: 5 body: { "text": "amsterdam" } - do: index: index: test - type: _doc id: 6 body: { "text": "amsterdam" } diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yml index 7ce6b86c6b525..dd29e7701ba1c 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yml @@ -3,7 +3,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: @@ -42,7 +41,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: @@ -103,7 +101,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: @@ -112,7 +109,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test2" } @@ -162,7 +158,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: @@ -171,7 +166,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test2" } @@ -259,13 +253,11 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: index: index: twitter - type: _doc id: 2 body: { "user": "junk" } - do: @@ -296,13 +288,11 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: index: index: twitter - type: _doc id: 2 body: { "user": "kimchy" } - do: @@ -343,17 +333,14 @@ - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: indices.refresh: {} diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yml index 7be77132b5910..3caa8c4b43405 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yml @@ -19,7 +19,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: @@ -36,7 +35,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: @@ -53,7 +51,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/40_versioning.yml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/40_versioning.yml index 8448b229b258b..8832b6a65c3dd 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/40_versioning.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/40_versioning.yml @@ -7,7 +7,6 @@ - do: index: index: index1 - type: _doc id: 1 version: 0 # Starting version is zero version_type: external @@ -28,6 +27,5 @@ - do: get: index: index1 - type: _doc id: 1 - match: {_version: 0} diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/50_wait_for_active_shards.yml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/50_wait_for_active_shards.yml index 9e7d9c4298a2f..ea8ed4df3e748 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/50_wait_for_active_shards.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/50_wait_for_active_shards.yml @@ -9,7 +9,6 @@ - do: index: index: test - type: _doc id: 1 body: {"text": "test"} - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/70_throttle.yml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/70_throttle.yml index d02985e13bc40..ad400ec718c87 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/70_throttle.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/70_throttle.yml @@ -10,17 +10,14 @@ - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: indices.refresh: {} @@ -50,17 +47,14 @@ - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: indices.refresh: {} @@ -91,17 +85,14 @@ - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: indices.refresh: {} @@ -151,17 +142,14 @@ - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: indices.refresh: {} diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/80_slices.yml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/80_slices.yml index d92910af00c66..f4a1a9805632a 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/80_slices.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/80_slices.yml @@ -3,25 +3,21 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: index: index: test - type: _doc id: 2 body: { "text": "test" } - do: index: index: test - type: _doc id: 3 body: { "text": "test" } - do: index: index: test - type: _doc id: 4 body: { "text": "test" } - do: @@ -69,25 +65,21 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: index: index: test - type: _doc id: 2 body: { "text": "test" } - do: index: index: test - type: _doc id: 3 body: { "text": "test" } - do: index: index: test - type: _doc id: 4 body: { "text": "test" } - do: @@ -176,37 +168,31 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: index: index: test - type: _doc id: 2 body: { "text": "test" } - do: index: index: test - type: _doc id: 3 body: { "text": "test" } - do: index: index: test - type: _doc id: 4 body: { "text": "test" } - do: index: index: test - type: _doc id: 5 body: { "text": "test" } - do: index: index: test - type: _doc id: 6 body: { "text": "test" } - do: @@ -297,25 +283,21 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: index: index: test - type: _doc id: 2 body: { "text": "test" } - do: index: index: test - type: _doc id: 3 body: { "text": "test" } - do: index: index: test - type: _doc id: 4 body: { "text": "test" } - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yml index 9a8e46e9abf76..312a88ace5e92 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yml @@ -3,7 +3,6 @@ - do: index: index: source - type: _doc id: 1 body: { "text": "test" } - do: @@ -38,13 +37,11 @@ - do: index: index: source - type: _doc id: 1 body: { "text": "test" } - do: index: index: dest - type: _doc id: 1 body: { "text": "test" } - do: @@ -79,7 +76,6 @@ - do: index: index: source - type: _doc id: 1 body: { "text": "test" } - do: @@ -136,13 +132,11 @@ - do: index: index: source - type: _doc id: 1 body: { "text": "test" } - do: index: index: dest - type: _doc id: 1 body: { "text": "test" } - do: @@ -183,13 +177,11 @@ - do: index: index: source - type: _doc id: 1 body: { "text": "test" } - do: index: index: dest - type: _doc id: 1 body: { "text": "test" } - do: @@ -224,7 +216,6 @@ - do: index: index: source - type: _doc id: 1 body: {} - do: @@ -242,7 +233,6 @@ - do: get: index: dest - type: _doc id: 1 - match: { _source: {} } @@ -258,7 +248,6 @@ - do: index: index: source - type: foo id: 1 body: { "text": "test", "filtered": "removed" } refresh: true @@ -287,7 +276,6 @@ - do: get: index: dest - type: _doc id: 1 - match: { _source.text: "test" } - is_false: _source.filtered diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yml index bef31b1bd799c..f726b1f00bddf 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yml @@ -44,7 +44,6 @@ - do: index: index: source - type: foo id: 1 body: { "text": "test" } - do: @@ -100,7 +99,6 @@ - do: index: index: test - type: test id: 1 body: { "text": "test" } - do: @@ -174,7 +172,6 @@ - do: index: index: test - type: test id: 1 body: { age: 23 } - do: @@ -301,7 +298,6 @@ - do: index: index: source - type: foo id: 1 body: { "text": "test" } - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/25_no_auto_create.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/25_no_auto_create.yml index 961084a5c04cb..d7fb98549f88f 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/25_no_auto_create.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/25_no_auto_create.yml @@ -10,7 +10,6 @@ teardown: - do: index: index: test - type: test id: 1 body: { "text": "test" } - do: @@ -37,7 +36,6 @@ teardown: - do: index: index: test - type: test id: 1 body: { "text": "test" } - do: @@ -59,7 +57,6 @@ teardown: - do: index: index: test - type: test id: 1 body: { "text": "test" } - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/30_search.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/30_search.yml index 61ae775b20895..dff24e4413368 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/30_search.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/30_search.yml @@ -3,13 +3,11 @@ - do: index: index: test - type: test id: 1 body: { "text": "test" } - do: index: index: test - type: test id: 2 body: { "text": "junk" } - do: @@ -38,13 +36,11 @@ - do: index: index: test - type: test id: 1 body: { "order": 1 } - do: index: index: test - type: test id: 2 body: { "order": 2 } - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/35_search_failures.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/35_search_failures.yml index 89135449d6969..5fd888f77a119 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/35_search_failures.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/35_search_failures.yml @@ -10,7 +10,6 @@ - do: index: index: source - type: foo id: 1 body: { "text": "test" } - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/40_versioning.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/40_versioning.yml index 4a53814b982ac..3d718831187b4 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/40_versioning.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/40_versioning.yml @@ -7,7 +7,6 @@ - do: index: index: src - type: test id: 1 body: { "company": "cat" } version: 2 @@ -15,13 +14,11 @@ - do: index: index: src - type: test id: 2 body: { "company": "cow" } - do: index: index: dest - type: test id: 1 body: { "company": "dog" } - do: @@ -54,7 +51,6 @@ - do: index: index: src - type: test id: 1 body: { "company": "cat" } version: 2 @@ -62,13 +58,11 @@ - do: index: index: src - type: test id: 2 body: { "company": "cow" } - do: index: index: dest - type: test id: 1 body: { "company": "dog" } - do: @@ -103,7 +97,6 @@ - do: index: index: src - type: test id: 1 body: { "company": "cat" } version: 2 @@ -111,13 +104,11 @@ - do: index: index: src - type: test id: 2 body: { "company": "cow" } - do: index: index: dest - type: test id: 1 body: { "company": "dog" } - do: @@ -151,19 +142,16 @@ - do: index: index: src - type: test id: 1 body: { "company": "cat" } - do: index: index: src - type: test id: 2 body: { "company": "cow" } - do: index: index: dest - type: test id: 1 body: { "company": "dog" } - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/50_routing.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/50_routing.yml index 362acebcc33f9..d7a0db5451a1d 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/50_routing.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/50_routing.yml @@ -3,7 +3,6 @@ - do: index: index: src - type: _doc id: 1 body: { "company": "cat" } - do: @@ -22,7 +21,6 @@ - do: get: index: dest - type: _doc id: 1 routing: cat - match: { _routing: cat } @@ -32,7 +30,6 @@ - do: index: index: src - type: _doc id: 1 body: { "company": "cat" } routing: null @@ -52,6 +49,5 @@ - do: get: index: dest - type: _doc id: 1 - is_false: _routing diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/60_wait_for_active_shards.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/60_wait_for_active_shards.yml index aee40112c7acb..3498e555d2879 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/60_wait_for_active_shards.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/60_wait_for_active_shards.yml @@ -9,7 +9,6 @@ - do: index: index: src - type: _doc id: 1 body: {"text": "test"} - do: @@ -43,5 +42,4 @@ - do: get: index: dest - type: _doc id: 1 diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/70_throttle.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/70_throttle.yml index 27cdecf93f4bf..696fdd068c454 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/70_throttle.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/70_throttle.yml @@ -12,19 +12,16 @@ - do: index: index: source - type: foo id: 1 body: { "text": "test" } - do: index: index: source - type: foo id: 2 body: { "text": "test" } - do: index: index: source - type: foo id: 3 body: { "text": "test" } - do: @@ -61,19 +58,16 @@ - do: index: index: source - type: foo id: 1 body: { "text": "test" } - do: index: index: source - type: foo id: 2 body: { "text": "test" } - do: index: index: source - type: foo id: 3 body: { "text": "test" } - do: @@ -110,19 +104,16 @@ - do: index: index: source - type: foo id: 1 body: { "text": "test" } - do: index: index: source - type: foo id: 2 body: { "text": "test" } - do: index: index: source - type: foo id: 3 body: { "text": "test" } - do: @@ -165,19 +156,16 @@ - do: index: index: source - type: foo id: 1 body: { "text": "test" } - do: index: index: source - type: foo id: 2 body: { "text": "test" } - do: index: index: source - type: foo id: 3 body: { "text": "test" } - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/80_slices.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/80_slices.yml index 7c4d24d917f3a..150ec2a4be45c 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/80_slices.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/80_slices.yml @@ -3,25 +3,21 @@ - do: index: index: source - type: foo id: 1 body: { "text": "test" } - do: index: index: source - type: foo id: 2 body: { "text": "test" } - do: index: index: source - type: foo id: 3 body: { "text": "test" } - do: index: index: source - type: foo id: 4 body: { "text": "test" } - do: @@ -65,25 +61,21 @@ - do: index: index: source - type: foo id: 1 body: { "text": "test" } - do: index: index: source - type: foo id: 2 body: { "text": "test" } - do: index: index: source - type: foo id: 3 body: { "text": "test" } - do: index: index: source - type: foo id: 4 body: { "text": "test" } - do: @@ -181,37 +173,31 @@ - do: index: index: source - type: foo id: 1 body: { "text": "test" } - do: index: index: source - type: foo id: 2 body: { "text": "test" } - do: index: index: source - type: foo id: 3 body: { "text": "test" } - do: index: index: source - type: foo id: 4 body: { "text": "test" } - do: index: index: source - type: foo id: 5 body: { "text": "test" } - do: index: index: source - type: foo id: 6 body: { "text": "test" } - do: @@ -306,25 +292,21 @@ - do: index: index: source - type: foo id: 1 body: { "text": "test" } - do: index: index: source - type: foo id: 2 body: { "text": "test" } - do: index: index: source - type: foo id: 3 body: { "text": "test" } - do: index: index: source - type: foo id: 4 body: { "text": "test" } - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/85_scripting.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/85_scripting.yml index 3f8a83c43fe19..ee920a669a504 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/85_scripting.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/85_scripting.yml @@ -3,7 +3,6 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: @@ -38,13 +37,11 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: index: index: twitter - type: _doc id: 2 body: { "user": "blort" } - do: @@ -89,13 +86,11 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: index: index: twitter - type: _doc id: 2 body: { "user": "foo" } - do: @@ -118,7 +113,6 @@ - do: get: index: new_twitter - type: _doc id: 1 routing: kimchy - match: { _routing: kimchy } @@ -126,7 +120,6 @@ - do: get: index: new_twitter - type: _doc id: 2 routing: foo - match: { _routing: foo } @@ -136,13 +129,11 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: index: index: twitter - type: _doc id: 2 body: { "user": "foo" } - do: @@ -192,13 +183,11 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: index: index: twitter - type: _doc id: 2 body: { "user": "foo" } - do: @@ -227,7 +216,6 @@ - do: index: index: twitter - type: _doc id: 1 version: 1 version_type: external @@ -235,7 +223,6 @@ - do: index: index: new_twitter - type: _doc id: 1 version: 1 version_type: external @@ -273,13 +260,11 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: index: index: new_twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: @@ -314,13 +299,11 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: index: index: twitter - type: _doc id: 2 body: { "user": "another" } - do: @@ -366,32 +349,27 @@ - do: index: index: index1 - type: _doc id: 1 body: { "lang": "en", "id": 123 } - do: index: index: index1 - type: _doc id: 2 body: { "lang": "en", "id": 456 } - do: index: index: index1 - type: _doc id: 3 body: { "lang": "fr", "id": 789 } # Destination index - do: index: index: index2 - type: _doc id: fr_789 body: { "lang": "fr", "id": 789 } - do: index: index: index2 - type: _doc id: en_123 body: { "lang": "en", "id": 123 } - do: @@ -444,7 +422,6 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/90_remote.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/90_remote.yml index 4e0282b8fabc4..65d6438e75d28 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/90_remote.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/90_remote.yml @@ -3,7 +3,6 @@ - do: index: index: source - type: _doc id: 1 body: { "text": "test" } refresh: true @@ -59,13 +58,11 @@ - do: index: index: source - type: _doc id: 1 body: { "text": "test" } - do: index: index: source - type: _doc id: 2 body: { "text": "test2" } - do: @@ -116,7 +113,6 @@ - do: index: index: source - type: _doc id: 1 body: { "text": "test" } routing: foo @@ -169,7 +165,6 @@ - do: index: index: source - type: _doc id: 1 body: { "text": "test" } refresh: true @@ -227,14 +222,12 @@ - do: index: index: source - type: _doc id: 1 body: { "text": "test" } refresh: true - do: index: index: source - type: _doc id: 2 body: { "text": "test" } refresh: true @@ -291,7 +284,6 @@ - do: index: index: source - type: _doc id: 1 body: { "text": "test" } refresh: true @@ -323,7 +315,6 @@ - do: index: index: source - type: _doc id: 1 body: { "text": "test" } refresh: true @@ -345,7 +336,6 @@ - do: index: index: source - type: _doc id: 1 body: { "text": "test", "filtered": "removed" } refresh: true @@ -385,7 +375,6 @@ - do: get: index: dest - type: _doc id: 1 - match: { _source.text: "test" } - is_false: _source.filtered @@ -404,19 +393,16 @@ - do: index: index: source - type: _doc id: 1 body: { "text": "test" } - do: index: index: source - type: _doc id: 2 body: { "text": "test" } - do: index: index: source - type: _doc id: 3 body: { "text": "test" } - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/95_parent_join.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/95_parent_join.yml index 24f263c4210fc..ae571d97c105a 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/95_parent_join.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/95_parent_join.yml @@ -20,14 +20,12 @@ setup: - do: index: index: source - type: doc id: 1 body: { "join_field": { "name": "parent" } } - do: index: index: source - type: doc id: 2 routing: 1 body: { "join_field": { "name": "child", "parent": "1" } } @@ -35,7 +33,6 @@ setup: - do: index: index: source - type: doc id: 3 routing: 1 body: { "join_field": { "name": "grand_child", "parent": "2" } } diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yml index eb76681c0b0d1..15bc62214ebfb 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yml @@ -3,7 +3,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: @@ -30,7 +29,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: @@ -87,7 +85,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: @@ -96,7 +93,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test2" } @@ -132,7 +128,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: @@ -141,7 +136,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test2" } @@ -203,13 +197,11 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: index: index: twitter - type: _doc id: 2 body: { "user": "junk" } - do: @@ -233,13 +225,11 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: index: index: twitter - type: _doc id: 2 body: { "user": "kimchy" } - do: @@ -267,17 +257,14 @@ - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: indices.refresh: {} @@ -293,7 +280,6 @@ - do: index: index: test - type: _doc id: 1 body: {} - do: @@ -307,7 +293,6 @@ - do: get: index: test - type: _doc id: 1 - match: { _source: {} } - match: { _version: 2 } diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yml index 8fdce4bd9db99..5e750f89f12ad 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yml @@ -3,7 +3,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: @@ -17,7 +16,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: @@ -31,7 +29,6 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: @@ -53,7 +50,6 @@ - do: index: index: test - type: _doc id: 1 body: { age: 23 } - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/30_new_fields.yml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/30_new_fields.yml index d20208f68e229..aba57218b8a3b 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/30_new_fields.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/30_new_fields.yml @@ -12,7 +12,6 @@ - do: index: index: test - type: _doc id: 1 refresh: true body: { "name": "bob! house" } diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/35_search_failure.yml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/35_search_failure.yml index 631f06a78476e..da94187f50afa 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/35_search_failure.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/35_search_failure.yml @@ -10,7 +10,6 @@ - do: index: index: source - type: _doc id: 1 body: { "text": "test" } - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/40_versioning.yml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/40_versioning.yml index d14691be53b83..3aa6c0918800d 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/40_versioning.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/40_versioning.yml @@ -3,7 +3,6 @@ - do: index: index: test - type: _doc id: 1 body: {"text": "test"} - do: @@ -18,7 +17,6 @@ - do: get: index: test - type: _doc id: 1 - match: {_version: 2} @@ -30,7 +28,6 @@ - do: index: index: index1 - type: _doc id: 1 version: 0 # Starting version is zero version_type: external @@ -48,6 +45,5 @@ - do: get: index: index1 - type: _doc id: 1 - match: {_version: 0} diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/50_consistency.yml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/50_consistency.yml index bb6d5abf2a7af..4a067580b54d3 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/50_consistency.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/50_consistency.yml @@ -9,7 +9,6 @@ - do: index: index: test - type: _doc id: 1 body: {"text": "test"} - do: @@ -35,5 +34,4 @@ - do: get: index: test - type: _doc id: 1 diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/60_throttle.yml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/60_throttle.yml index 2153508e88d32..a8491280a7069 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/60_throttle.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/60_throttle.yml @@ -10,17 +10,14 @@ - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: indices.refresh: {} @@ -46,17 +43,14 @@ - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: indices.refresh: {} @@ -83,17 +77,14 @@ - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: indices.refresh: {} @@ -130,17 +121,14 @@ - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: index: index: test - type: _doc body: { "text": "test" } - do: indices.refresh: {} diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/70_slices.yml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/70_slices.yml index 4474eaece9727..3e8d82f13d36c 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/70_slices.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/70_slices.yml @@ -3,25 +3,21 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: index: index: test - type: _doc id: 2 body: { "text": "test" } - do: index: index: test - type: _doc id: 3 body: { "text": "test" } - do: index: index: test - type: _doc id: 4 body: { "text": "test" } - do: @@ -61,25 +57,21 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: index: index: test - type: _doc id: 2 body: { "text": "test" } - do: index: index: test - type: _doc id: 3 body: { "text": "test" } - do: index: index: test - type: _doc id: 4 body: { "text": "test" } - do: @@ -163,37 +155,31 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: index: index: test - type: _doc id: 2 body: { "text": "test" } - do: index: index: test - type: _doc id: 3 body: { "text": "test" } - do: index: index: test - type: _doc id: 4 body: { "text": "test" } - do: index: index: test - type: _doc id: 5 body: { "text": "test" } - do: index: index: test - type: _doc id: 6 body: { "text": "test" } - do: @@ -280,25 +266,21 @@ - do: index: index: test - type: _doc id: 1 body: { "text": "test" } - do: index: index: test - type: _doc id: 2 body: { "text": "test" } - do: index: index: test - type: _doc id: 3 body: { "text": "test" } - do: index: index: test - type: _doc id: 4 body: { "text": "test" } - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/80_scripting.yml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/80_scripting.yml index 6ce30c6036287..0c297b13dbd81 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/80_scripting.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/80_scripting.yml @@ -3,7 +3,6 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: @@ -35,7 +34,6 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: @@ -64,13 +62,11 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: index: index: twitter - type: _doc id: 2 body: { "user": "foo" } - do: @@ -112,13 +108,11 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: index: index: twitter - type: _doc id: 2 body: { "user": "foo" } - do: @@ -141,7 +135,6 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: @@ -161,7 +154,6 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: @@ -181,25 +173,21 @@ - do: index: index: twitter - type: _doc id: 1 body: { "level": 9, "last_updated": "2016-01-01T12:10:30Z" } - do: index: index: twitter - type: _doc id: 2 body: { "level": 10, "last_updated": "2016-01-01T12:10:30Z" } - do: index: index: twitter - type: _doc id: 3 body: { "level": 11, "last_updated": "2016-01-01T12:10:30Z" } - do: index: index: twitter - type: _doc id: 4 body: { "level": 12, "last_updated": "2016-01-01T12:10:30Z" } - do: @@ -247,25 +235,21 @@ - do: index: index: twitter - type: _doc id: 1 body: { "level": 9, "last_updated": "2016-01-01T12:10:30Z" } - do: index: index: twitter - type: _doc id: 2 body: { "level": 10, "last_updated": "2016-01-01T12:10:30Z" } - do: index: index: twitter - type: _doc id: 3 body: { "level": 11, "last_updated": "2016-01-01T12:10:30Z" } - do: index: index: twitter - type: _doc id: 4 body: { "level": 12, "last_updated": "2016-01-01T12:10:30Z" } - do: @@ -326,13 +310,11 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: index: index: twitter - type: _doc id: 2 body: { "user": "foo" } - do: @@ -355,25 +337,21 @@ - do: index: index: twitter - type: _doc id: 1 body: { "level": 9, "last_updated": "2016-01-01T12:10:30Z" } - do: index: index: twitter - type: _doc id: 2 body: { "level": 10, "last_updated": "2016-01-01T12:10:30Z" } - do: index: index: twitter - type: _doc id: 3 body: { "level": 11, "last_updated": "2016-01-01T12:10:30Z" } - do: index: index: twitter - type: _doc id: 4 body: { "level": 12, "last_updated": "2016-01-01T12:10:30Z" } - do: @@ -439,7 +417,6 @@ - do: index: index: twitter - type: _doc id: 1 body: { "user": "kimchy" } - do: diff --git a/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yml b/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yml index 1adbfc73bc7b8..b932f0d53caad 100644 --- a/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yml +++ b/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yml @@ -7,7 +7,7 @@ # snapshots. In order to do that it uses a URLFixture that exposes # the content of the shared directory over HTTP. A second URL # repository is used to test the snapshot restore but this time -# with a "file://" prefix. +# with a "file://" prefix. setup: # Ensure that the FS repository is registered, so we can create @@ -23,17 +23,14 @@ setup: body: - index: _index: docs - _type: doc _id: 1 - snapshot: one - index: _index: docs - _type: doc _id: 2 - snapshot: one - index: _index: docs - _type: doc _id: 3 - snapshot: one @@ -51,22 +48,18 @@ setup: body: - index: _index: docs - _type: doc _id: 4 - snapshot: two - index: _index: docs - _type: doc _id: 5 - snapshot: two - index: _index: docs - _type: doc _id: 6 - snapshot: two - index: _index: docs - _type: doc _id: 7 - snapshot: two diff --git a/plugins/analysis-icu/src/test/resources/rest-api-spec/test/analysis_icu/20_search.yml b/plugins/analysis-icu/src/test/resources/rest-api-spec/test/analysis_icu/20_search.yml index d739db2ff72e6..aaa3ea7fb042b 100644 --- a/plugins/analysis-icu/src/test/resources/rest-api-spec/test/analysis_icu/20_search.yml +++ b/plugins/analysis-icu/src/test/resources/rest-api-spec/test/analysis_icu/20_search.yml @@ -4,6 +4,7 @@ "Index ICU content": - do: indices.create: + include_type_name: false index: test body: settings: @@ -19,16 +20,14 @@ language: en strength: primary mappings: - type: - properties: - text: - type: text - analyzer: my_analyzer + properties: + text: + type: text + analyzer: my_analyzer - do: index: index: test - type: type id: 1 body: { "text": "Bâton enflammé" } - do: diff --git a/plugins/analysis-kuromoji/src/test/resources/rest-api-spec/test/analysis_nori/20_search.yml b/plugins/analysis-kuromoji/src/test/resources/rest-api-spec/test/analysis_nori/20_search.yml index 729c5b418e1c8..3b7fc4eb46293 100644 --- a/plugins/analysis-kuromoji/src/test/resources/rest-api-spec/test/analysis_nori/20_search.yml +++ b/plugins/analysis-kuromoji/src/test/resources/rest-api-spec/test/analysis_nori/20_search.yml @@ -16,7 +16,6 @@ - do: index: index: test - type: type id: 1 body: { "text": "JR新宿駅の近くにビールを飲みに行こうか" } - do: diff --git a/plugins/analysis-nori/src/test/resources/rest-api-spec/test/analysis_nori/20_search.yml b/plugins/analysis-nori/src/test/resources/rest-api-spec/test/analysis_nori/20_search.yml index 0f4752ba62af5..e31355d299e9a 100644 --- a/plugins/analysis-nori/src/test/resources/rest-api-spec/test/analysis_nori/20_search.yml +++ b/plugins/analysis-nori/src/test/resources/rest-api-spec/test/analysis_nori/20_search.yml @@ -16,7 +16,6 @@ - do: index: index: test - type: type id: 1 body: { "text": "뿌리가 깊은 나무는" } - do: diff --git a/plugins/analysis-phonetic/src/test/resources/rest-api-spec/test/analysis_phonetic/40_search.yml b/plugins/analysis-phonetic/src/test/resources/rest-api-spec/test/analysis_phonetic/40_search.yml index 98e9df0c861f7..80f7b63bb0b7c 100644 --- a/plugins/analysis-phonetic/src/test/resources/rest-api-spec/test/analysis_phonetic/40_search.yml +++ b/plugins/analysis-phonetic/src/test/resources/rest-api-spec/test/analysis_phonetic/40_search.yml @@ -4,6 +4,7 @@ "Index phonetic content": - do: indices.create: + include_type_name: false index: phonetic_sample body: settings: @@ -19,16 +20,14 @@ encoder: metaphone replace: false mappings: - type: - properties: - text: - type: text - analyzer: my_analyzer + properties: + text: + type: text + analyzer: my_analyzer - do: index: index: phonetic_sample - type: type id: 1 body: { "text": "hello world" } - do: diff --git a/plugins/analysis-smartcn/src/test/resources/rest-api-spec/test/analysis_smartcn/20_search.yml b/plugins/analysis-smartcn/src/test/resources/rest-api-spec/test/analysis_smartcn/20_search.yml index 4c343c3a7ec7e..2529e40c0be5e 100644 --- a/plugins/analysis-smartcn/src/test/resources/rest-api-spec/test/analysis_smartcn/20_search.yml +++ b/plugins/analysis-smartcn/src/test/resources/rest-api-spec/test/analysis_smartcn/20_search.yml @@ -4,19 +4,18 @@ "Index Smartcn content": - do: indices.create: + include_type_name: false index: test body: mappings: - type: - properties: - text: - type: text - analyzer: smartcn + properties: + text: + type: text + analyzer: smartcn - do: index: index: test - type: type id: 1 body: { "text": "我购买了道具和服装" } - do: diff --git a/plugins/analysis-stempel/src/test/resources/rest-api-spec/test/analysis_stempel/20_search.yml b/plugins/analysis-stempel/src/test/resources/rest-api-spec/test/analysis_stempel/20_search.yml index 153f3528e7dc9..dc00c297694db 100644 --- a/plugins/analysis-stempel/src/test/resources/rest-api-spec/test/analysis_stempel/20_search.yml +++ b/plugins/analysis-stempel/src/test/resources/rest-api-spec/test/analysis_stempel/20_search.yml @@ -4,19 +4,18 @@ "Index Stempel content": - do: indices.create: + include_type_name: false index: test body: mappings: - type: - properties: - text: - type: text - analyzer: polish + properties: + text: + type: text + analyzer: polish - do: index: index: test - type: type id: 1 body: { "text": "studenta był" } - do: diff --git a/plugins/analysis-ukrainian/src/test/resources/rest-api-spec/test/analysis_ukrainian/20_search.yml b/plugins/analysis-ukrainian/src/test/resources/rest-api-spec/test/analysis_ukrainian/20_search.yml index fca9711a85570..aec452ed3fe99 100644 --- a/plugins/analysis-ukrainian/src/test/resources/rest-api-spec/test/analysis_ukrainian/20_search.yml +++ b/plugins/analysis-ukrainian/src/test/resources/rest-api-spec/test/analysis_ukrainian/20_search.yml @@ -4,19 +4,18 @@ "Index Stempel content": - do: indices.create: + include_type_name: false index: test body: mappings: - type: - properties: - text: - type: text - analyzer: ukrainian + properties: + text: + type: text + analyzer: ukrainian - do: index: index: test - type: type id: 1 body: { "text": "Ця п'єса у свою чергу рухається по колу." } - do: diff --git a/plugins/examples/custom-suggester/src/test/resources/rest-api-spec/test/custom-suggester/20_suggest.yml b/plugins/examples/custom-suggester/src/test/resources/rest-api-spec/test/custom-suggester/20_suggest.yml index 8e56a00454514..993dd86ade2af 100644 --- a/plugins/examples/custom-suggester/src/test/resources/rest-api-spec/test/custom-suggester/20_suggest.yml +++ b/plugins/examples/custom-suggester/src/test/resources/rest-api-spec/test/custom-suggester/20_suggest.yml @@ -24,7 +24,6 @@ - do: bulk: index: test - type: test refresh: true body: | { "index": {} } diff --git a/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/20_whitelist.yml b/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/20_whitelist.yml index 9cd5f781aab51..51a440142fd5e 100644 --- a/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/20_whitelist.yml +++ b/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/20_whitelist.yml @@ -4,7 +4,6 @@ - do: index: index: test - type: test id: 1 body: { "num1": 1.0 } - do: diff --git a/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/30_static.yml b/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/30_static.yml index e55fdb457486e..c6d8048b97961 100644 --- a/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/30_static.yml +++ b/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/30_static.yml @@ -4,7 +4,6 @@ - do: index: index: test - type: test id: 1 body: { "num1": 1 } - do: diff --git a/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/40_instance.yml b/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/40_instance.yml index edad5bc9993ee..385d576ae48e9 100644 --- a/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/40_instance.yml +++ b/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/40_instance.yml @@ -4,7 +4,6 @@ - do: index: index: test - type: test id: 1 body: { "num1": 1 } - do: diff --git a/plugins/examples/rescore/src/test/resources/rest-api-spec/test/example-rescore/20_score.yml b/plugins/examples/rescore/src/test/resources/rest-api-spec/test/example-rescore/20_score.yml index 68710e8b383e8..bcdc05b4c8612 100644 --- a/plugins/examples/rescore/src/test/resources/rest-api-spec/test/example-rescore/20_score.yml +++ b/plugins/examples/rescore/src/test/resources/rest-api-spec/test/example-rescore/20_score.yml @@ -11,13 +11,11 @@ setup: - do: index: index: test - type: test id: 1 body: { "test": 1 } - do: index: index: test - type: test id: 2 body: { "test": 2 } - do: diff --git a/plugins/examples/script-expert-scoring/src/test/resources/rest-api-spec/test/script_expert_scoring/20_score.yml b/plugins/examples/script-expert-scoring/src/test/resources/rest-api-spec/test/script_expert_scoring/20_score.yml index 4f0ef22fc8426..c771ba82312a6 100644 --- a/plugins/examples/script-expert-scoring/src/test/resources/rest-api-spec/test/script_expert_scoring/20_score.yml +++ b/plugins/examples/script-expert-scoring/src/test/resources/rest-api-spec/test/script_expert_scoring/20_score.yml @@ -9,19 +9,16 @@ setup: - do: index: index: test - type: test id: 1 body: { "important_field": "foo" } - do: index: index: test - type: test id: 2 body: { "important_field": "foo foo foo" } - do: index: index: test - type: test id: 3 body: { "important_field": "foo foo" } diff --git a/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/20_attachment_processor.yml b/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/20_attachment_processor.yml index 6a22071ba3829..5aba14690ee18 100644 --- a/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/20_attachment_processor.yml +++ b/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/20_attachment_processor.yml @@ -19,7 +19,6 @@ - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: { field1: "VGhpcyBpcyBhbiBlbmdsaXNoIHRleHQgdG8gdGVzdCBpZiB0aGUgcGlwZWxpbmUgd29ya3M=" } @@ -27,7 +26,6 @@ - do: get: index: test - type: test id: 1 - match: { _source.field1: "VGhpcyBpcyBhbiBlbmdsaXNoIHRleHQgdG8gdGVzdCBpZiB0aGUgcGlwZWxpbmUgd29ya3M=" } - length: { _source.attachment: 4 } @@ -62,7 +60,6 @@ - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: { field1: "VGhpcyBpcyBhbiBlbmdsaXNoIHRleHQgdG8gdGVzdCBpZiB0aGUgcGlwZWxpbmUgd29ya3MK" } @@ -70,7 +67,6 @@ - do: get: index: test - type: test id: 1 - match: { _source.field1: "VGhpcyBpcyBhbiBlbmdsaXNoIHRleHQgdG8gdGVzdCBpZiB0aGUgcGlwZWxpbmUgd29ya3MK" } - length: { _source.attachment: 1 } @@ -98,7 +94,6 @@ - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: { field1: "VGhpcyBpcyBhbiBlbmdsaXNoIHRleHQgdG8gdGVzdCBpZiB0aGUgcGlwZWxpbmUgd29ya3M=" } @@ -106,7 +101,6 @@ - do: get: index: test - type: test id: 1 - length: { _source.attachment: 4 } - match: { _source.attachment.content: "This is an english text to tes" } @@ -136,7 +130,6 @@ - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: { field1: "VGhpcyBpcyBhbiBlbmdsaXNoIHRleHQgdG8gdGVzdCBpZiB0aGUgcGlwZWxpbmUgd29ya3M=" } @@ -144,7 +137,6 @@ - do: get: index: test - type: test id: 1 - length: { _source.attachment: 4 } - match: { _source.attachment.content: "This is an english text to tes" } @@ -154,7 +146,6 @@ - do: index: index: test - type: test id: 2 pipeline: "my_pipeline" body: { field1: "VGhpcyBpcyBhbiBlbmdsaXNoIHRleHQgdG8gdGVzdCBpZiB0aGUgcGlwZWxpbmUgd29ya3M=", "max_size": 18 } @@ -162,7 +153,6 @@ - do: get: index: test - type: test id: 2 - length: { _source.attachment: 4 } - match: { _source.attachment.content: "This is an english" } @@ -172,7 +162,6 @@ - do: index: index: test - type: test id: 3 pipeline: "my_pipeline" body: { field1: "VGhpcyBpcyBhbiBlbmdsaXNoIHRleHQgdG8gdGVzdCBpZiB0aGUgcGlwZWxpbmUgd29ya3M=", "max_size": 100000000 } @@ -180,7 +169,6 @@ - do: get: index: test - type: test id: 3 - length: { _source.attachment: 4 } - match: { _source.attachment.content: "This is an english text to test if the pipeline works" } diff --git a/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/30_files_supported.yml b/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/30_files_supported.yml index c8ab6f0ba9d34..543f394782da9 100644 --- a/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/30_files_supported.yml +++ b/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/30_files_supported.yml @@ -19,7 +19,6 @@ - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: { field1: "0M8R4KGxGuEAAAAAAAAAAAAAAAAAAAAAPgADAP7/CQAGAAAAAAAAAAAAAAAEAAAAjAEAAAAAAAAAEAAAjgEAAAEAAAD+////AAAAAIgBAACJAQAAigEAAIsBAAD////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////spcEAg+kMBAAA8BK/AAAAAAABEQABAAEACAAAEwgAAA4AYmpiaoI4gjgAAAAAAAAAAAAAAAAAAAAAAAAMBBYANA4AAOBSAADgUgAAEwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD//w8AAAAAAAAAAAD//w8AAAAAAAAAAAD//w8AAAAAAAAAAAAAAAAAAAAAALcAAAAAAFAHAAAAAAAAUAcAAMcUAAAAAAAAxxQAAAAAAADHFAAAAAAAAMcUAAAAAAAAxxQAABQAAAAAAAAAAAAAAP////8AAAAA2xQAAAAAAADbFAAAAAAAANsUAAAAAAAA2xQAAAwAAADnFAAADAAAANsUAAAAAAAA3hUAADABAADzFAAAAAAAAPMUAAAAAAAA8xQAAAAAAADzFAAAAAAAAPMUAAAAAAAA8xQAAAAAAADzFAAAAAAAAPMUAAAAAAAAVRUAAAIAAABXFQAAAAAAAFcVAAAAAAAAVxUAAAAAAABXFQAAAAAAAFcVAAAAAAAAVxUAACwAAAAOFwAAtgIAAMQZAABaAAAAgxUAABUAAAAAAAAAAAAAAAAAAAAAAAAAxxQAAAAAAADzFAAAAAAAAAAAAAAAAAAAAAAAAAAAAADzFAAAAAAAAPMUAAAAAAAA8xQAAAAAAADzFAAAAAAAAIMVAAAAAAAAGRUAAAAAAADHFAAAAAAAAMcUAAAAAAAA8xQAAAAAAAAAAAAAAAAAAPMUAAAAAAAAmBUAABYAAAAZFQAAAAAAABkVAAAAAAAAGRUAAAAAAADzFAAAFgAAAMcUAAAAAAAA8xQAAAAAAADHFAAAAAAAAPMUAAAAAAAAVRUAAAAAAAAAAAAAAAAAABkVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA8xQAAAAAAABVFQAAAAAAAAAAAAAAAAAAGRUAAAAAAAAAAAAAAAAAABkVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGRUAAAAAAAAAAAAAAAAAAP////8AAAAAgI6XYKZ60QEAAAAAAAAAAP////8AAAAACRUAABAAAAAZFQAAAAAAAAAAAAAAAAAAQRUAABQAAACuFQAAMAAAAN4VAAAAAAAAGRUAAAAAAAAeGgAAAAAAABkVAAAAAAAAHhoAAAAAAAAZFQAAAAAAABkVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADHFAAAAAAAABkVAAAoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA8xQAAAAAAADzFAAAAAAAAPMUAAAAAAAAgxUAAAAAAACDFQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGRUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPMUAAAAAAAA8xQAAAAAAADzFAAAAAAAAN4VAAAAAAAA8xQAAAAAAADzFAAAAAAAAPMUAAAAAAAA8xQAAAAAAAAAAAAAAAAAAP////8AAAAA/////wAAAAD/////AAAAAAAAAAAAAAAA/////wAAAAD/////AAAAAP////8AAAAA/////wAAAAD/////AAAAAP////8AAAAA/////wAAAAD/////AAAAAP////8AAAAA/////wAAAAD/////AAAAAP////8AAAAA/////wAAAAD/////AAAAAB4aAAAAAAAA8xQAAAAAAADzFAAAAAAAAPMUAAAAAAAA8xQAAAAAAADzFAAAAAAAAPMUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADzFAAAAAAAAPMUAAAAAAAA8xQAAAAAAABQBwAAPQwAAI0TAAA6AQAABwAMAQ8ADQEAAAwEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFRlc3QgZWxhc3RpY3NlYXJjaA0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAABIIAAATCAAA/PgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAYWaJVGuQAABhZo3wiGAAIACAAAEwgAAP0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAATIAMZBoATpwpBeqAB+wfC4gsMhBIbCJBSKwiQUjkIkFJJCJBSWwAAAXsMQCGLDEAgyQxAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAALgYPABIAAQB8AQ8ACAADAAMAAwAAAAQACAAAAJgAAACeAAAAngAAAJ4AAACeAAAAngAAAJ4AAACeAAAAngAAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAAHYCAAB2AgAAdgIAAHYCAAB2AgAAdgIAAHYCAAB2AgAAdgIAADYGAAA2BgAANgYAADYGAAA2BgAANgYAAD4CAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAACoAAAANgYAADYGAAAWAAAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAC4AAAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAAaAEAAEgBAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAAHACAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAAMgYAABgAAADGAwAA1gMAAOYDAAD2AwAABgQAABYEAAAmBAAANgQAAEYEAABWBAAAZgQAAHYEAACGBAAAlgQAAMYDAADWAwAA5gMAAPYDAAAGBAAAFgQAADIGAAAoAgAA2AEAAOgBAAAmBAAANgQAAEYEAABWBAAAZgQAAHYEAACGBAAAlgQAAMYDAADWAwAA5gMAAPYDAAAGBAAAFgQAACYEAAA2BAAARgQAAFYEAABmBAAAdgQAAIYEAACWBAAAxgMAANYDAADmAwAA9gMAAAYEAAAWBAAAJgQAADYEAABGBAAAVgQAAGYEAAB2BAAAhgQAAJYEAADGAwAA1gMAAOYDAAD2AwAABgQAABYEAAAmBAAANgQAAEYEAABWBAAAZgQAAHYEAACGBAAAlgQAAMYDAADWAwAA5gMAAPYDAAAGBAAAFgQAACYEAAA2BAAARgQAAFYEAABmBAAAdgQAAIYEAACWBAAAxgMAANYDAADmAwAA9gMAAAYEAAAWBAAAJgQAADYEAABGBAAAVgQAAGYEAAB2BAAAhgQAAJYEAAA4AQAAWAEAAPgBAAAIAgAAGAIAAFYCAAB+AgAAkAIAAKACAACwAgAAwAIAANACAACAAgAA4AIAAPACAAAAAwAAEAMAACADAAAwAwAAQAMAAOACAADwAgAAAAMAABADAAAgAwAAMAMAAEADAADgAgAA8AIAAAADAAAQAwAAIAMAADADAABAAwAA4AIAAPACAAAAAwAAEAMAACADAAAwAwAAQAMAAOACAADwAgAAAAMAABADAAAgAwAAMAMAAEADAADgAgAA8AIAAAADAAAQAwAAIAMAADADAABAAwAA4AIAAPACAAAAAwAAEAMAACADAAAwAwAAQAMAAOACAADwAgAAAAMAABADAAAgAwAAMAMAAEADAADgAgAA8AIAAAADAAAQAwAAIAMAADADAABAAwAA4AIAAPACAAAAAwAAEAMAACADAAAwAwAAQAMAAOACAADwAgAAAAMAABADAAAgAwAAMAMAAEADAADgAgAA8AIAAAADAAAQAwAAIAMAADADAABAAwAA4AIAAPACAAAAAwAAEAMAACADAAAwAwAAQAMAAOACAADwAgAAAAMAABADAAAgAwAAMAMAAEADAAAgAAAAT0oDAFBKAwBRSgMAX0gBBG1IDARuSAwEc0gMBHRIDAQAAAAAQAAAYPH/AgBAAAwQAAAAAAAAAAAGAE4AbwByAG0AYQBsAAAAAgAAABgAQ0oYAF9IAQRhShgAbUgMBHNIDAR0SAkEAAAAAAAAAAAAAAAAAAAAAAAAOgBBIPL/oQA6AAwNAAAAAAAAEAARAFAAbwBsAGkAYwBlACAAcABhAHIAIABkAOkAZgBhAHUAdAAAAAAAVgBpAPP/swBWAAwNAAAAAAAAMAYOAFQAYQBiAGwAZQBhAHUAIABOAG8AcgBtAGEAbAAAABwAF/YDAAA01gYAAQoDbAA01gYAAQUDAABh9gMAAAIACwAAADIAayD0/8EAMgAADQAAAAAAADAGDABBAHUAYwB1AG4AZQAgAGwAaQBzAHQAZQAAAAIADAAAAAAAUEsDBBQABgAIAAAAIQCb6HBP/AAAABwCAAATAAAAW0NvbnRlbnRfVHlwZXNdLnhtbKyRy2rDMBBF94X+g9C22HK6KKXYzqKPXR+L9AMGeWyL2CMhTULy9x07LpQSAoVuBNLMvffMqFwfxkHtMSbnqdKrvNAKyfrGUVfpz81Ldq9VYqAGBk9Y6SMmva6vr8rNMWBSoqZU6Z45PBiTbI8jpNwHJKm0Po7Aco2dCWC30KG5LYo7Yz0xEmc8eei6fMIWdgOr54M8n0hErtXjqW+KqjSEMDgLLKBmqpqzuohDuiDcU/OLLlvIclHO5ql3Id0sCe+ymugaVB8Q+Q1G4TAsQ+LP8xVIRov5ZeYz0b5tncXG290o68hn48XsTwCr/4n+zjTz39ZfAAAA//8DAFBLAwQUAAYACAAAACEApdan58AAAAA2AQAACwAAAF9yZWxzLy5yZWxzhI/PasMwDIfvhb2D0X1R0sMYJXYvpZBDL6N9AOEof2giG9sb69tPxwYKuwiEpO/3qT3+rov54ZTnIBaaqgbD4kM/y2jhdj2/f4LJhaSnJQhbeHCGo3vbtV+8UNGjPM0xG6VItjCVEg+I2U+8Uq5CZNHJENJKRds0YiR/p5FxX9cfmJ4Z4DZM0/UWUtc3YK6PqMn/s8MwzJ5PwX+vLOVFBG43lExp5GKhqC/jU72QqGWq1B7Qtbj51v0BAAD//wMAUEsDBBQABgAIAAAAIQBreZYWgwAAAIoAAAAcAAAAdGhlbWUvdGhlbWUvdGhlbWVNYW5hZ2VyLnhtbAzMTQrDIBBA4X2hd5DZN2O7KEVissuuu/YAQ5waQceg0p/b1+XjgzfO3xTVm0sNWSycBw2KZc0uiLfwfCynG6jaSBzFLGzhxxXm6XgYybSNE99JyHNRfSPVkIWttd0g1rUr1SHvLN1euSRqPYtHV+jT9yniResrJgoCOP0BAAD//wMAUEsDBBQABgAIAAAAIQBtTVmryAYAAI4aAAAWAAAAdGhlbWUvdGhlbWUvdGhlbWUxLnhtbOxZ3YrbRhS+L/QdhO4d/0n+WeINtmxv2uwmIXbS5nJWHkuTHWmMZrwbEwJ9gkIhLb0p9K6F3gTaN+i7pLTpQ/TMSJZn7HH2hy2E0jUs8vg7Z7455+g7I83dey8T6pzjjBOW9tz6nZrr4DRkM5JGPffpdFzpuA4XKJ0hylLcc1eYu/cOP/3kLjoQMU6wA/YpP0A9NxZicVCt8hCGEb/DFjiF3+YsS5CAr1lUnWXoAvwmtNqo1VrVBJHUdVKUgNtp/PvP4OzRfE5C7B6uvY8oTJEKLgdCmk2kb1yYDJYZRkuFnZ3VJYKveEAz5xzRngsTzdjFFL8UrkMRF/BDz62pP7d6eLeKDgojKvbYanZj9VfYFQazs4aaM4tOy0k9z/da/dK/AlCxixu1R61Rq/SnACgMYaU5F92nP+gOhn6B1UD5pcX3sD1s1g285r+5w7nvy4+BV6Dcv7eDH48DiKKBV6Ac7+/gPa/dCDwDr0A5vrWDb9f6Q69t4BUopiQ920HX/FYzWK+2hMwZvW+Fd31v3G4UzjcoqIayuuQUc5aKfbWWoBcsGwNAAikSJHXEaoHnKIQyDhAlpxlxjkkUQ+EtUMo4DNcatXGtCf/lx1NXKiLoACPNWvICJnxnSPJxeJiRhei5n4NXV4M8XzpHTMQkLGZVTgyL+yiNdIv3P33z9w9fOX/9+uP7N9/mk27juY4f4jT6kqD0QxPAajdhePfd2z9+e/vu+6///OWNxX8/Q6c6fEoSzJ2H+MJ5whJYnGUF+DS7nsU0RkS36KcRRymSs1j8jyB+OvrhClFkwQ0gEjruWQYyYwMeLV8YhCdxthTE4vFBnBjAE8bogGXWKDyQc2lhni7TyD55ttRxTxA6t80doNTI82i5AH0lNpdBjA2ajylKBYpwioUjf2NnGFtW95wQI64nJMwYZ3PhPCfOABFrSKbk1KimjdF9kkBeVjaCkG8jNifPnAGjtlUP8bmJhLsDUQv5KaZGGI/QUqDE5nKKEqoH/BiJ2EZysspCHTfiAjIdYcqc0QxzbrN5lMF6taQ/AImxp/2ErhITmQlyZvN5jBjTkUN2FsQoWdiwE5LGOvYzfgYlipzHTNjgJ8y8Q+R3yAOIx750PyPYSPflavAU1FWntCkQ+csys+TyCDOjficrOkdYSQ2Iv6HpCUkvFfgtaff/PWk/IWkYM8uKbkvU7a6NjFxTzvsZsd5P97dEfB9uW7oDls3Ix6/cQ7RMH2O4WXbb1//C/b9wu/954d53P9++XG8UGsRbbl3zzbrauid7d+5zQulErCg+5mrzzqEvzcYwKO3UYysun+QWMVzKOxkmMHBRhpSNkzHxBRHxJEYL2OHXXekk4oXriDsLxmHjr4atviWeLpMTNssfWOt1+XCaiwdHYjNe88txeNgQObrV3jyEle4V20g9LK8JSNvrkNAmM0k0LSTa60EZJPVoDkGzkFAruxUWXQuLjnS/TtUOC6BWZgU2Tg5st3qu74EJGMEzFaJ4JvOUp3qdXZXM28z0vmAaFQC7iHUFbDLdlVz3Lk+uLi+1K2TaIKGVm0lCRUb1MB6jGS6qU45ehcZ1c93dpNSgJ0Oh5oPS2tBodz7E4qa5BrttbaCprhQ0dS56bqvpQ8mEaNFz5/DgD5fJAmqHyw0vohG8PgtFlt/wN1GWRcbFEPE4D7gSnVwNEiJw5lCS9Fy5/DINNFUaorjVGyAIHy25LsjKx0YOkm4mGc/nOBR62rURGen8Kyh8rhXWX5X5zcHSki0h3ZN4duGc0mX2BEGJ+e26DOCMcHj/U8+jOSPwQrMUsk39bTWmQnb1N4qqhvJxRBcxKjqKLuY5XEl5SUd9K2OgfSvWDAHVQlI0wtNINlg9qEY3LbtGzmFv173cSEZOE81NzzRURXZNu4oZM6zbwFYsb9bkNVbrEIOm6R0+l+5tye2utW5rn1B2CQh4GT9L171CQ9CobSYzqEnGuzIsNbsYNXvHeoGXULtKk9BUv7V2uxW3skdYp4PBG3V+sNuuWhiar/eVKtLq6EM/nGCnL0A8hvAaeEkFV6mEo4cMwYZoovYkuWzALfJSFLcGXDnLjPTcVzW/7wUNP6jUOv6o4jW9WqXj95uVvu836yO/XhsOGq+hsYg4qfv5scsYXkTRVXH4osZ3DmCS9bu2OyFLqkydrFQVcXUAU28YBzD5yYszlQcsrkNAdF61GuNusztoVbrN/rjiDQedSjdoDSrDVtAejoeB3+mOX7vOuQJ7/WbgtUadSqseBBWvVZP0O91K22s0+l673xl5/dfFNgZWnstHEQsIr+J1+A8AAAD//wMAUEsDBBQABgAIAAAAIQAN0ZCftgAAABsBAAAnAAAAdGhlbWUvdGhlbWUvX3JlbHMvdGhlbWVNYW5hZ2VyLnhtbC5yZWxzhI9NCsIwFIT3gncIb2/TuhCRJt2I0K3UA4TkNQ02PyRR7O0NriwILodhvplpu5edyRNjMt4xaKoaCDrplXGawW247I5AUhZOidk7ZLBggo5vN+0VZ5FLKE0mJFIoLjGYcg4nSpOc0IpU+YCuOKOPVuQio6ZByLvQSPd1faDxmwF8xSS9YhB71QAZllCa/7P9OBqJZy8fFl3+UUFz2YUFKKLGzOAjm6pMBMpburrE3wAAAP//AwBQSwECLQAUAAYACAAAACEAm+hwT/wAAAAcAgAAEwAAAAAAAAAAAAAAAAAAAAAAW0NvbnRlbnRfVHlwZXNdLnhtbFBLAQItABQABgAIAAAAIQCl1qfnwAAAADYBAAALAAAAAAAAAAAAAAAAAC0BAABfcmVscy8ucmVsc1BLAQItABQABgAIAAAAIQBreZYWgwAAAIoAAAAcAAAAAAAAAAAAAAAAABYCAAB0aGVtZS90aGVtZS90aGVtZU1hbmFnZXIueG1sUEsBAi0AFAAGAAgAAAAhAG1NWavIBgAAjhoAABYAAAAAAAAAAAAAAAAA0wIAAHRoZW1lL3RoZW1lL3RoZW1lMS54bWxQSwECLQAUAAYACAAAACEADdGQn7YAAAAbAQAAJwAAAAAAAAAAAAAAAADPCQAAdGhlbWUvdGhlbWUvX3JlbHMvdGhlbWVNYW5hZ2VyLnhtbC5yZWxzUEsFBgAAAAAFAAUAXQEAAMoKAAAAADw/eG1sIHZlcnNpb249IjEuMCIgZW5jb2Rpbmc9IlVURi04IiBzdGFuZGFsb25lPSJ5ZXMiPz4NCjxhOmNsck1hcCB4bWxuczphPSJodHRwOi8vc2NoZW1hcy5vcGVueG1sZm9ybWF0cy5vcmcvZHJhd2luZ21sLzIwMDYvbWFpbiIgYmcxPSJsdDEiIHR4MT0iZGsxIiBiZzI9Imx0MiIgdHgyPSJkazIiIGFjY2VudDE9ImFjY2VudDEiIGFjY2VudDI9ImFjY2VudDIiIGFjY2VudDM9ImFjY2VudDMiIGFjY2VudDQ9ImFjY2VudDQiIGFjY2VudDU9ImFjY2VudDUiIGFjY2VudDY9ImFjY2VudDYiIGhsaW5rPSJobGluayIgZm9sSGxpbms9ImZvbEhsaW5rIi8+AAAAABMAAAAUAAAOAAAIAP////8ACAAAEwgAAAUAAAAACAAAEwgAAAYAAAAAAAAABQAAABIAAAAVAAAABwAEAAcAAAAAABIAAAAVAAAABAAHAAQAAAAEAAAACAAAAOUAAAAAAAAAAwAAAN8IhgCkF6oAlUa5AH419AAAAAAAEwAAABUAAAAAAAAAAQAAAP9AAIABABIAAAASAAAAAEBDewEAAQASAAAAAAAAABIAAAAAAAAAAAAAAAAAAAACEAAAAAAAAAATAAAAoAAAEABAAAD//wEAAAAHAFUAbgBrAG4AbwB3AG4A//8BAAgAAAAAAAAAAAAAAP//AQAAAAAA//8AAAIA//8AAAAA//8AAAIA//8AAAAABQAAAEcOkAEAAAICBgMFBAUCAwTvKgDgQXgAwAkAAAAAAAAA/wEAAAAAAABUAGkAbQBlAHMAIABOAGUAdwAgAFIAbwBtAGEAbgAAADUOkAECAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAgAAAAABTAHkAbQBiAG8AbAAAADMOkAEAAAILBgQCAgICAgT/KgDgQ3gAwAkAAAAAAAAA/wEAAAAAAABBAHIAaQBhAGwAAAA3DpABAAACDwUCAgIEAwIE/wIA4P+sAEABAAAAAAAAAJ8BAAAAAAAAQwBhAGwAaQBiAHIAaQAAAEESkAEBAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABDAGEAbQBiAHIAaQBhACAATQBhAHQAaAAAACAABADxCIgIAPDEAgAAqQEAAAAAWVJDh1lSQ4cAAAAAAgABAAAAAgAAABEAAAABAAEAAAAEAAOQAQAAAAIAAAARAAAAAQABAAAAAQAAAAAAAAAhAwDwEAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAClBsAHtAC0AIGBcjAAAAAAAAAAAAAAAAAAABIAAAASAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAABAAAAA8BAACAD8/QEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACSFAAAAAACfH/DwAAJFAAABAnAAD///9/////f////3////9/////f////3////9/3wiGAAAEAAAyAAAAAAAAAAAAAAAAAAAAAAAAAAAAIQQAAAAAAAAAAAAAAAAAAAAAAAAQHAAABAAAAAAAAAAAAHgAAAB4AAAAAAAAAAAAAACgBQAAGkjOCAsAAAAAAAAA3AAAAAEAAAD//xIAAAAAAAAAAAAAAAAAAAAMAEQAYQB2AGkAZAAgAFAAaQBsAGEAdABvAAwARABhAHYAaQBkACAAUABpAGwAYQB0AG8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAP7/AAADCgEAAAAAAAAAAAAAAAAAAAAAAAEAAADghZ/y+U9oEKuRCAArJ7PZMAAAANzSAgASAAAAAQAAAJgAAAACAAAAoAAAAAMAAACsAAAABAAAALgAAAAFAAAA0AAAAAYAAADcAAAABwAAAOgAAAAIAAAA/AAAAAkAAAAUAQAAEgAAACABAAAKAAAARAEAAAwAAABQAQAADQAAAFwBAAAOAAAAaAEAAA8AAABwAQAAEAAAAHgBAAATAAAAgAEAABEAAACIAQAAAgAAABAnAAAeAAAABAAAAAAAAAAeAAAABAAAAAAAAAAeAAAAEAAAAERhdmlkIFBpbGF0bwAAAAAeAAAABAAAAAAAAAAeAAAABAAAAAAAAAAeAAAADAAAAE5vcm1hbC5kb3RtAB4AAAAQAAAARGF2aWQgUGlsYXRvAAAAAB4AAAAEAAAAMgAAAB4AAAAcAAAATWljcm9zb2Z0IE1hY2ludG9zaCBXb3JkAAAAAEAAAAAARsMjAAAAAEAAAAAAFjZWpnrRAUAAAAAAFjZWpnrRAQMAAAABAAAAAwAAAAIAAAADAAAAEQAAAAMAAAAAAAAARwAAAEzRAgD/////DgAAAAEAAABsAAAAAAAAAAAAAAD/AAAAswAAAAAAAAAAAAAAZhkAANsRAAAgRU1GAAABAETRAgAIAAAAAQAAAAAAAAAAAAAAAAAAAOwEAACxAwAAQAEAAPAAAAAAAAAAAAAAAAAAAAAA4gQAgKkDABEAAAAMAAAACAAAAAoAAAAQAAAAAAAAAAAAAAAJAAAAEAAAAAABAAC0AAAADAAAABAAAAAAAAAAAAAAAAsAAAAQAAAAAAEAALQAAABRAAAAeNACAAAAAAAAAAAA/wAAALMAAAAAAAAAAAAAAAAAAAAAAAAAAAEAALQAAABQAAAAKAAAAHgAAAAA0AIAAAAAACAAzAAAAQAAtAAAACgAAAAAAQAAtAAAAAEAIAAAAAAAANACAAAAAAAAAAAAAAAAAAAAAAD/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////vr6+/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/76+vv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////7vf//+rz7v/Yzc3/0NLY/+DX2f/N4PL/3tXI/8jV4v/Q0cX/1tDI/9ve2f/U0tX/0NLQ/83I0P/I2N7/4tnI/9LZ4v/v6tz/5eXl////9////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////83g9//e3M3/vrG3/8TCxv/Xwrz/vdfu/8W/rv/K1tX/x8bB/8LJxv/Oxb7/yMTE/8vCwv+3scH/zd7Z/9DNyP/BwcT/z97X/82xq/////v////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////u9/v/+/Lu////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////++vr7/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/vr6+//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8OAAAAFAAAAAAAAAAQAAAAFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD+/wAAAwoBAAAAAAAAAAAAAAAAAAAAAAABAAAAAtXN1ZwuGxCTlwgAKyz5rjAAAADUAAAACwAAAAEAAABgAAAABQAAAGgAAAAGAAAAcAAAABEAAAB4AAAAFwAAAIAAAAALAAAAiAAAABAAAACQAAAAEwAAAJgAAAAWAAAAoAAAAA0AAACoAAAADAAAALUAAAACAAAAECcAAAMAAAABAAAAAwAAAAEAAAADAAAAEgAAAAMAAAAAAA8ACwAAAAAAAAALAAAAAAAAAAsAAAAAAAAACwAAAAAAAAAeEAAAAQAAAAEAAAAADBAAAAIAAAAeAAAABgAAAFRpdHJlAAMAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAIAAAADAAAABAAAAAUAAAAGAAAABwAAAP7///8JAAAACgAAAAsAAAAMAAAADQAAAA4AAAAPAAAAEAAAABEAAAASAAAAEwAAABQAAAAVAAAA/v///xcAAAAYAAAAGQAAABoAAAAbAAAAHAAAAB0AAAAeAAAAHwAAACAAAAAhAAAAIgAAACMAAAAkAAAAJQAAACYAAAAnAAAAKAAAACkAAAAqAAAAKwAAACwAAAAtAAAALgAAAC8AAAAwAAAAMQAAADIAAAAzAAAANAAAADUAAAA2AAAANwAAADgAAAA5AAAAOgAAADsAAAA8AAAAPQAAAD4AAAA/AAAAQAAAAEEAAABCAAAAQwAAAEQAAABFAAAARgAAAEcAAABIAAAASQAAAEoAAABLAAAATAAAAE0AAABOAAAATwAAAFAAAABRAAAAUgAAAFMAAABUAAAAVQAAAFYAAABXAAAAWAAAAFkAAABaAAAAWwAAAFwAAABdAAAAXgAAAF8AAABgAAAAYQAAAGIAAABjAAAAZAAAAGUAAABmAAAAZwAAAGgAAABpAAAAagAAAGsAAABsAAAAbQAAAG4AAABvAAAAcAAAAHEAAAByAAAAcwAAAHQAAAB1AAAAdgAAAHcAAAB4AAAAeQAAAHoAAAB7AAAAfAAAAH0AAAB+AAAAfwAAAIAAAACBAAAAggAAAIMAAACEAAAAhQAAAIYAAACHAAAAiAAAAIkAAACKAAAAiwAAAIwAAACNAAAAjgAAAI8AAACQAAAAkQAAAJIAAACTAAAAlAAAAJUAAACWAAAAlwAAAJgAAACZAAAAmgAAAJsAAACcAAAAnQAAAJ4AAACfAAAAoAAAAKEAAACiAAAAowAAAKQAAAClAAAApgAAAKcAAACoAAAAqQAAAKoAAACrAAAArAAAAK0AAACuAAAArwAAALAAAACxAAAAsgAAALMAAAC0AAAAtQAAALYAAAC3AAAAuAAAALkAAAC6AAAAuwAAALwAAAC9AAAAvgAAAL8AAADAAAAAwQAAAMIAAADDAAAAxAAAAMUAAADGAAAAxwAAAMgAAADJAAAAygAAAMsAAADMAAAAzQAAAM4AAADPAAAA0AAAANEAAADSAAAA0wAAANQAAADVAAAA1gAAANcAAADYAAAA2QAAANoAAADbAAAA3AAAAN0AAADeAAAA3wAAAOAAAADhAAAA4gAAAOMAAADkAAAA5QAAAOYAAADnAAAA6AAAAOkAAADqAAAA6wAAAOwAAADtAAAA7gAAAO8AAADwAAAA8QAAAPIAAADzAAAA9AAAAPUAAAD2AAAA9wAAAPgAAAD5AAAA+gAAAPsAAAD8AAAA/QAAAP4AAAD/AAAAAAEAAAEBAAACAQAAAwEAAAQBAAAFAQAABgEAAAcBAAAIAQAACQEAAAoBAAALAQAADAEAAA0BAAAOAQAADwEAABABAAARAQAAEgEAABMBAAAUAQAAFQEAABYBAAAXAQAAGAEAABkBAAAaAQAAGwEAABwBAAAdAQAAHgEAAB8BAAAgAQAAIQEAACIBAAAjAQAAJAEAACUBAAAmAQAAJwEAACgBAAApAQAAKgEAACsBAAAsAQAALQEAAC4BAAAvAQAAMAEAADEBAAAyAQAAMwEAADQBAAA1AQAANgEAADcBAAA4AQAAOQEAADoBAAA7AQAAPAEAAD0BAAA+AQAAPwEAAEABAABBAQAAQgEAAEMBAABEAQAARQEAAEYBAABHAQAASAEAAEkBAABKAQAASwEAAEwBAABNAQAATgEAAE8BAABQAQAAUQEAAFIBAABTAQAAVAEAAFUBAABWAQAAVwEAAFgBAABZAQAAWgEAAFsBAABcAQAAXQEAAF4BAABfAQAAYAEAAGEBAABiAQAAYwEAAGQBAABlAQAAZgEAAGcBAABoAQAAaQEAAGoBAABrAQAAbAEAAG0BAABuAQAAbwEAAHABAABxAQAAcgEAAHMBAAB0AQAAdQEAAHYBAAB3AQAAeAEAAHkBAAB6AQAAewEAAHwBAAB9AQAAfgEAAH8BAAD+////gQEAAIIBAACDAQAAhAEAAIUBAACGAQAAhwEAAP7////9/////f////3////9////jQEAAP7////+/////v////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////9SAG8AbwB0ACAARQBuAHQAcgB5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFgAFAf//////////AwAAAAYJAgAAAAAAwAAAAAAAAEYAAAAAAAAAAAAAAAAgFZlgpnrRAY8BAACAAAAAAAAAADEAVABhAGIAbABlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOAAIB/////wUAAAD/////AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAB4aAAAAAAAAVwBvAHIAZABEAG8AYwB1AG0AZQBuAHQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABoAAgEBAAAA//////////8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAFAFMAdQBtAG0AYQByAHkASQBuAGYAbwByAG0AYQB0AGkAbwBuAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAACAQIAAAAEAAAA/////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABYAAAAM0wIAAAAAAAUARABvAGMAdQBtAGUAbgB0AFMAdQBtAG0AYQByAHkASQBuAGYAbwByAG0AYQB0AGkAbwBuAAAAAAAAAAAAAAA4AAIB////////////////AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAEAAAAQAAAAAAAAAQBDAG8AbQBwAE8AYgBqAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABIAAgD///////////////8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAcgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAP///////////////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA////////////////AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAP7///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8BAP7/AwoAAP////8GCQIAAAAAAMAAAAAAAABGIAAAAERvY3VtZW50IE1pY3Jvc29mdCBXb3JkIDk3LTIwMDQACgAAAE1TV29yZERvYwAQAAAAV29yZC5Eb2N1bWVudC44APQ5snEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" } @@ -27,7 +26,6 @@ - do: get: index: test - type: test id: 1 - length: { _source.attachment: 6 } - match: { _source.attachment.content: "Test elasticsearch" } @@ -59,7 +57,6 @@ - do: index: index: test - type: test id: 1 pipeline: "my_pipeline" body: { field1: "UEsDBBQABgAIAAAAIQBtiidLZgEAAFQFAAATAAgCW0NvbnRlbnRfVHlwZXNdLnhtbCCiBAIooAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC0lMtugzAQRfeV+g/I2wqcdFFVVUgWfSzbSE0/wLEH4tYv2c7r7ztAgqooAalJNkgwc+89A3hGk41WyQp8kNbkZJgNSAKGWyFNmZOv2Vv6SJIQmRFMWQM52UIgk/HtzWi2dRASVJuQk0WM7onSwBegWcisA4OVwnrNIt76kjrGf1gJ9H4weKDcmggmprHyIOPRCxRsqWLyusHHDQnKSfLc9FVROWHOKclZxDKtqvSozoMKHcKVEQd06Y4sQ2XdExbShbvTCd8OyoMEqavR6gJqPvB1eikgmTIf35nGBrq2XlBh+VKjKOse7gijLQrJodVXbs5bDiHgd9IqayuaSbNnP8kR4lZBuDxF49sfDzGi4BoAO+dehDXMP69G8ce8F6TA3BmbK7g8RmvdCxHx1EJzHZ7NUdt0RWLn1FsXcAv4f4y9P66VOsWBHfgou/+6NhGtz54Pqk0gQBzJpvVOHP8CAAD//wMAUEsDBBQABgAIAAAAIQDHwie8/wAAAN8CAAALAAgCX3JlbHMvLnJlbHMgogQCKKAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAArJLNSgMxEIDvgu8Q5t7NtoqINNuLCL2JrA8wJtPd6OaHZKrt2xtF1IVlEexx/j6+SWa9ObhBvFLKNngFy6oGQV4HY32n4LG9W1yDyIze4BA8KThShk1zfrZ+oAG5DOXexiwKxWcFPXO8kTLrnhzmKkTypbILySGXMHUyon7BjuSqrq9k+s2AZsQUW6Mgbc0FiPYY6X9s6YjRIKPUIdEipjKd2JZdRIupI1Zggr4v6fzZURUyyGmhy78Lhd3OaroNeu/I85QXHZi8ITOvhDHOGS1PaTTu+JF5C8lI85Wes1md9sO437snj3aYeJfvWvUcqfsQkqOzbN4BAAD//wMAUEsDBBQABgAIAAAAIQATqj6H9gAAADEDAAAcAAgBd29yZC9fcmVscy9kb2N1bWVudC54bWwucmVscyCiBAEooAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKySy2rDMBBF94X+g5h9LTt9UELkbEoh29b9AEUeP6gsCc304b+vaEjr0GC68PJeMfeeQbPZfg5WvGOk3jsFRZaDQGd83btWwUv1eHUPgli7WlvvUMGIBNvy8mLzhFZzGqKuDyRSiiMFHXNYS0mmw0FT5gO69NL4OGhOMrYyaPOqW5SrPL+TcZoB5Umm2NUK4q6+BlGNAf+T7ZumN/jgzduAjs9UyA/cPyNzWo5SrI4tsoKJmaVEkOdBbpYEabzjSu8t/mL8WHMQt0tCcJqdAHzLg1nMMRRLMhCPFiefcdBz9atF6/9cw9E5IsiTQy+/AAAA//8DAFBLAwQUAAYACAAAACEA9WKOYGUCAAAOBwAAEQAAAHdvcmQvZG9jdW1lbnQueG1spFXfb9owEH6ftP8h8jtNwijQiFDR0qI+TKpK9zwZx0ksYp9lGyj763dOIGSbVtGSh9j367vv7mJncvsmq2DLjRWgUhJfRSTgikEmVJGSH6+PvTEJrKMqoxUonpI9t+R2+vXLZJdkwDaSKxcghLLJTrOUlM7pJAwtK7mk9koKZsBC7q4YyBDyXDAe7sBkYT+Ko3qnDTBuLea7p2pLLTnASTgPTVJ23PajaIyyUC3Gv4xAc4XGHIykDkVTYIRZb3QPMTV1YiUq4fYea9jCbFOyMSo5YPRaHj4mQQLJVlZHZ3jPtyF6WI4R5hySTcj80PKaXmh4hYRB2VLoU98+i4bG8gjybsGdYnc6Hlw29LmhO1xOgOfQz5ogWTXM30eMozMm4iHaiHMo/JnzyKT78e0+15pOc+PrjwH0/wbQxWXDWRjY6BOauAztSa1bLH+VfADrMORuafYyMsuSajyBkiVPhQJDVxUywpEF2PXAf9ZkilfcCrK9XzWqB4mmhj5lKRmNhg/X9/GI1FrH31yjbR7UJnidZi8piaK7m8Hw5rpVzXlON5XzlvEwGs8f6yzGv9z0lVsX4JG2TjDLqWHlJPR6/65dVgBrf1ktHTUOIQVmjTy2ohLZ/1zAHWVrEnZ9H1TWeoY1lPZmy5l7Nv9nukS7185m8WjW9EIXy19oxdMRxzdRnbfE/XA8qJG9w3fqIR3gIY4HdX8SI4rSncQVOAfyJFc871hLTjOO1+EoGnsxB3Adsdi4WjykY1BZ1FpNGW98ajX+lRZG+KIrofizcAxZfhseq28Kr7fNcMPTj2z6GwAA//8DAFBLAwQUAAYACAAAACEAbU1ZqyEGAACOGgAAFQAAAHdvcmQvdGhlbWUvdGhlbWUxLnhtbOxZy47bNhTdF+g/ENo7lm3Jj0E8gS3bSZuZJMg4abOkJVpihhINkpoZIwjQLyhQIC26KdBdC3QToP2D/kuKNv2IUpRlkzbdQToOEBSxAYuPcy8P7yUPJev2nauUgAvEOKZZ32ncch2AspBGOIv7zpPppNZ1ABcwiyChGeo7S8SdO8effnIbHokEpQhI+4wfwb6TCLE4qtd5KJshv0UXKJN9c8pSKGSVxfWIwUvpNyX1puu26ynEmQMymEq30+T3n6Wzh/M5DpFzXHkfE/mTCV40hISdFb7RymSYMwRzhY3OG8WFL3lAGLiApO/IgSJ6OUVXwgEEciE7+o6rPk79+HZ9bUTEHlvNbqI+K7uVQXTeVHYsnq0NPc/32oO1fwUgYhc37ozb4/banwLAMJQzLbnoWH/YG478FVYDlUWL71Fn1GoYeM1/awc/8IuvgVegsujt4CeTYBNDDVQWfUtMOs3AM/AKVBbbO/iOOxh5HQOvQAnB2fkO2vXbraCa7Royp+SeFd7zvUmnuYJvUHVtdZX2mdi31lL4nLKJBKjkQoEzIJYLNIehxAWQ4BnD4ATHiVx4C5hRLpvdpjtxW/K3+HqqpCICjxDUrMumkO80FXwADxleiL7zufTqaJBnObhLRYLD1ai7FvdgFusWb3/65u8fvgJ//frj21ff2vFcx49QFn+JYfZvAwjd4M13r//47fWb77/+85dXFviAwZkOn+IUcfAAXYLHNJWTswyAZuzdLKYJxLrFIIs5zGBhY0GPZfx09IMlJNCCGyIzkk+ZlAob8G7+3CB8lrBcYAvwfpIawFNKyZAy65zuF2PpUciz2D44y3XcYwgvbGMHW3ke5wu55rHNZZAgg+YjIlMOY5QhAYo+eo6QxewZxkZcT3HIKKdzAZ5hMITYGpIpnhmraWN0D6cyL0sbQZlvIzanT8GQEpv7EbowkXJ3QGJziYgRxrswFzC1MoYp0ZEnUCQ2kmdLFhoB50JmOkaEgnGEOLfZPGRLg+59KTH2tJ+SZWoimcDnNuQJpNTY4PQ8SGC6sHLGWaJjP+PncolC8IgKKwlq7pCiLvMgxWNfup9iZKT7+r39RMqQfYEUPTmzbQlEzf24JHOIlPP6lqanOLtW4Lek3X9/0n6KszChds09iKjboTeR8wHD1v20LeL7cNvSHVAW4Q9fuUcwzx4huVks0I/C/VG4//fCvW8/H16uNwqtbuOrm3XlJt175z7HhJyJJUEnXGk7l9OLJrJRVZTR+kFhkcjiajgDFzOoyoBR8QUWyVkCF3KYhhoh5ivXMQcLyuXpoJqtvosOkqenNCpbG43q2VQaQLFpl6dL1S7PIlG2tjubh7C1e1WL1cNyRaCwfRcS2mAmiZaFRKdqvIaEmtlBWPQsLLqF+70s1GWVFbn/ACz+1/C9kpFcb5CgqMhTaV9l9+CZ3hdMc9pNy/R6BdfDZNogoS03k4S2DBMYoe3mA+e6t0mpQa8IxS6NTvd95LoQkS1tIJlZA5dyz7V86SaEi74zl/eFspgupD9e6CYkcdZ3QrEK9H9RlgXjYgR5UsJUVzn/FAvEAMGpXOt6Gki24dZodoo5fqDkeu6HFzl10ZOM5nMUij0tm6rsK51Ye28ILio0l6TPkugSzEjOHkMZKL/TKAIYYS7W0Yww0xb3JopbcrXaisZ/ZpstCskigasTRRfzEq7KazraPBTT7VmZ9dVkZnGRpBufutcbFR2aaO45QIpT064f7++Q11htdN9gVUr3ttb1Kq3bd0rc/EDQqG0GM6gVjC3UNq0mtQPeEGjDrZfmvjPi0KfB9qotDojqvlLVdl5O0NlzufJH8nY1J4IrquhKPiME1d/KpRKo1kpdrgTIGe47L1x/4AVNP6i5XX9c81qeW+v6g1Zt4PutxthvuKNh86UMikjShl+OPZHPM2S5evmi2ndewKTVbfatkKZ1qt6s1JWxegHTaBovYMo3L2Ba9DsAy8i8aDcnvVZv2K71WoNJzRsNu7Ve0B7WRu2gM5qMAr/bm7x0wIUCe4NW4LXH3Vq7EQQ1r+0W9Lu9WsdrNgdeZ9Ade4OXq1jLmVfXKryK1/E/AAAA//8DAFBLAwQKAAAAAAAAACEAvOgH/fQnAAD0JwAAFwAAAGRvY1Byb3BzL3RodW1ibmFpbC5qcGVn/9j/4AAQSkZJRgABAQAASABIAAD/4QCARXhpZgAATU0AKgAAAAgABAEaAAUAAAABAAAAPgEbAAUAAAABAAAARgEoAAMAAAABAAIAAIdpAAQAAAABAAAATgAAAAAAAABIAAAAAQAAAEgAAAABAAOgAQADAAAAAQABAACgAgAEAAAAAQAAAWmgAwAEAAAAAQAAAgAAAAAA/+0AOFBob3Rvc2hvcCAzLjAAOEJJTQQEAAAAAAAAOEJJTQQlAAAAAAAQ1B2M2Y8AsgTpgAmY7PhCfv/AABEIAgABaQMBEQACEQEDEQH/xAAfAAABBQEBAQEBAQAAAAAAAAAAAQIDBAUGBwgJCgv/xAC1EAACAQMDAgQDBQUEBAAAAX0BAgMABBEFEiExQQYTUWEHInEUMoGRoQgjQrHBFVLR8CQzYnKCCQoWFxgZGiUmJygpKjQ1Njc4OTpDREVGR0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4eLj5OXm5+jp6vHy8/T19vf4+fr/xAAfAQADAQEBAQEBAQEBAAAAAAAAAQIDBAUGBwgJCgv/xAC1EQACAQIEBAMEBwUEBAABAncAAQIDEQQFITEGEkFRB2FxEyIygQgUQpGhscEJIzNS8BVictEKFiQ04SXxFxgZGiYnKCkqNTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqCg4SFhoeIiYqSk5SVlpeYmZqio6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2dri4+Tl5ufo6ery8/T19vf4+fr/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/3QAEAC7/2gAMAwEAAhEDEQA/AP7Yfgx8GPg9N8HvhRLL8KPhrLLL8NfAskkkngTws8kkj+F9LZ3d200s7uxLMzHczEk5JNAHpX/ClPg3/wBEl+GX/hBeFf8A5W0AH/ClPg3/ANEl+GX/AIQXhX/5W0AH/ClPg3/0SX4Zf+EF4V/+VtAB/wAKU+Df/RJfhl/4QXhX/wCVtAB/wpT4N/8ARJfhl/4QXhX/AOVtAB/wpT4N/wDRJfhl/wCEF4V/+VtAB/wpT4N/9El+GX/hBeFf/lbQAf8AClPg3/0SX4Zf+EF4V/8AlbQAf8KU+Df/AESX4Zf+EF4V/wDlbQAf8KU+Df8A0SX4Zf8AhBeFf/lbQAf8KU+Df/RJfhl/4QXhX/5W0AH/AApT4N/9El+GX/hBeFf/AJW0AH/ClPg3/wBEl+GX/hBeFf8A5W0AH/ClPg3/ANEl+GX/AIQXhX/5W0AH/ClPg3/0SX4Zf+EF4V/+VtAB/wAKU+Df/RJfhl/4QXhX/wCVtAB/wpT4N/8ARJfhl/4QXhX/AOVtAB/wpT4N/wDRJfhl/wCEF4V/+VtAB/wpT4N/9El+GX/hBeFf/lbQAf8AClPg3/0SX4Zf+EF4V/8AlbQAf8KU+Df/AESX4Zf+EF4V/wDlbQAf8KU+Df8A0SX4Zf8AhBeFf/lbQAf8KU+Df/RJfhl/4QXhX/5W0AH/AApT4N/9El+GX/hBeFf/AJW0AH/ClPg3/wBEl+GX/hBeFf8A5W0AH/ClPg3/ANEl+GX/AIQXhX/5W0AH/ClPg3/0SX4Zf+EF4V/+VtAB/wAKU+Df/RJfhl/4QXhX/wCVtAB/wpT4N/8ARJfhl/4QXhX/AOVtAB/wpT4N/wDRJfhl/wCEF4V/+VtAB/wpT4N/9El+GX/hBeFf/lbQAf8AClPg3/0SX4Zf+EF4V/8AlbQAf8KU+Df/AESX4Zf+EF4V/wDlbQAf8KU+Df8A0SX4Zf8AhBeFf/lbQAf8KU+Df/RJfhl/4QXhX/5W0AH/AApT4N/9El+GX/hBeFf/AJW0AH/ClPg3/wBEl+GX/hBeFf8A5W0AH/ClPg3/ANEl+GX/AIQXhX/5W0AH/ClPg3/0SX4Zf+EF4V/+VtAB/wAKU+Df/RJfhl/4QXhX/wCVtAB/wpT4N/8ARJfhl/4QXhX/AOVtAB/wpT4N/wDRJfhl/wCEF4V/+VtAB/wpT4N/9El+GX/hBeFf/lbQAf8AClPg3/0SX4Zf+EF4V/8AlbQB/Nd/wrT4c/8ARP8AwT/4Sug//INAH//Q/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9H+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0v7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/T/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9T+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAeDfEX4/+Hvhf8QfD3gbxN4W8Vx6ZrfgHxz8TNQ+JS33w9sPh34O8G/DOXRk8fav4uvfEHj3RPE1jD4Xt/Evhu/v307wrq0M1hrUU+ny3Z03Xk0oAxf8Ahrr9n+W68P2WneN7nXLrxJ4x0D4f2sHh7wj411x9L8aeI9T13RrPwx4qGmeHrr/hD9esNV8Na1YeJNG8Uto+q+Eriz2+KLLRxNbvKAXNd/am+C2h+MbHwIPFDa34kn8V6n4Q1Ox8OWU+sSaBf6R8P/iz8Q7+7v4YALrVNPt7T4K+PPCs58JW/ia/tfH2nf8ACHXun22rW2qRaWAZN7+2L+z3bRaLNY+N5/ECa/4p0XwRYyeG/DPinWLW38XeIPh9r/xR07w7rupW+jHSvC2qQ+B/Dt5r2vweJ77SB4Ns7zRZvGjeHoNc0qa6ANXSP2r/ANn7WUia2+JWhxg6X4r1q7mlNxLpWl6R4Asbe7+IWsaj4lsobvwrD4f+H91dQeHvGvimLXZ/C3hzxhJH4O1LWovE8sWkOANvf2s/2eNN06x1bUviZpmnadqdvey6fdahpXiSyS7v9N8cH4a6l4YgW50aGR/HenePlfwnf/D0L/wnNlrUctnc+HonikKgGaP2wfgC/wARfCHwzh8arNrHje68eaVoWsLYXkXhaTxJ8O/Gfw8+Hmu+GJtcuY4Ihqs/jj4m+HPB+l3MME+g3njFbzwS+tW/jP8As/QL8A+nKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoA/mXoA//1f7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQB5b8Sfgr8MPi/bahZ/EfwrB4otNV+HPxJ+Emo2t1qGsWltefDz4vW2gWfxF8OTw6ZqNlG0XiO18MaJDJf7Rq2mrZE6Pf6c11etcAHk2k/sUfs26Fq/hbXtJ8DavYaz4JstA07wrqNt8R/ifHc6LY+G/Gcfj/TbS0ZfGKqLZ/FKTX2p28ivBrFpqGs6LqsV3omu65p2oAEGtfsOfsueILvx9fal8MAbj4neIfEXivxo1j4y8f6Qmp+IvF3hL4ieCPE+rW0Ok+KLKDRbjXfDvxY+IUOprocenQXWpeJbrxBJEfEFtp+p2gBQ0H9gr9lfwv8AYz4f+HWraW2nW/hSx06S2+J3xZ86w0/wXbeLLLQ9MtLiTx01xBpf2Dx54107WdMSQWHiTT/FWvWXiKDVLbVLtJQB17+wd+y1qWhWfhXU/h9rOqeEdPvfH9/p3g7VPij8W9S8H6dc/FLRPEnh/wCIT6f4WvvHU2g2K+LNM8Y+K01aO10+FJbrxFrGoIqX17PcMAV9M/YB/ZM0jxR4H8Z2PwtuF8S/Di38NWvgzUZ/iF8T7saNF4R+Is3xY0DFjc+M5tNv5LP4gTvr8s2p2l5JfYTS79rnRkTT6AO9sf2UfgVp97a30PhPWJ5dP8S3vizR4NR+IPxI1XTvD+rah8V/AHxwuYvDek6n4vvNL8OaGPin8MPBPiu28LaJZ2Hhmxk0iXSbHSbfQNX1rS9SAPoqgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKAP5l6AP/W/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9f+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0P7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/R/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9L+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0/7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/U/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9X+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1v7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/X/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9D+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0f7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/S/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9P+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1P7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/V/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9b+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1/7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/Q/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9H+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0v7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/T/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9T+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1f7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/W/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9f+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0P7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/R/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9L+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0/7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/U/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9X+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1v7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/X/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9D+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0f7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/S/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9P+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1P7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/V/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9b+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1/7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/Q/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9H+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0v7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/T/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9T+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1f7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/W/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9f+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0P7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/R/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9L+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0/7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/U/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9X+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1v7Yfgx8Z/g9D8HvhRFL8V/hrFLF8NfAsckcnjvwskkcieF9LV0dG1IMjowKsrDcrAg4INAHpX/C6/g3/wBFa+GX/he+Ff8A5ZUAH/C6/g3/ANFa+GX/AIXvhX/5ZUAH/C6/g3/0Vr4Zf+F74V/+WVAB/wALr+Df/RWvhl/4XvhX/wCWVAB/wuv4N/8ARWvhl/4XvhX/AOWVAB/wuv4N/wDRWvhl/wCF74V/+WVAB/wuv4N/9Fa+GX/he+Ff/llQAf8AC6/g3/0Vr4Zf+F74V/8AllQAf8Lr+Df/AEVr4Zf+F74V/wDllQAf8Lr+Df8A0Vr4Zf8Ahe+Ff/llQAf8Lr+Df/RWvhl/4XvhX/5ZUAH/AAuv4N/9Fa+GX/he+Ff/AJZUAH/C6/g3/wBFa+GX/he+Ff8A5ZUAH/C6/g3/ANFa+GX/AIXvhX/5ZUAH/C6/g3/0Vr4Zf+F74V/+WVAB/wALr+Df/RWvhl/4XvhX/wCWVAB/wuv4N/8ARWvhl/4XvhX/AOWVAB/wuv4N/wDRWvhl/wCF74V/+WVAB/wuv4N/9Fa+GX/he+Ff/llQAf8AC6/g3/0Vr4Zf+F74V/8AllQAf8Lr+Df/AEVr4Zf+F74V/wDllQAf8Lr+Df8A0Vr4Zf8Ahe+Ff/llQAf8Lr+Df/RWvhl/4XvhX/5ZUAH/AAuv4N/9Fa+GX/he+Ff/AJZUAH/C6/g3/wBFa+GX/he+Ff8A5ZUAH/C6/g3/ANFa+GX/AIXvhX/5ZUAH/C6/g3/0Vr4Zf+F74V/+WVAB/wALr+Df/RWvhl/4XvhX/wCWVAB/wuv4N/8ARWvhl/4XvhX/AOWVAB/wuv4N/wDRWvhl/wCF74V/+WVAB/wuv4N/9Fa+GX/he+Ff/llQAf8AC6/g3/0Vr4Zf+F74V/8AllQAf8Lr+Df/AEVr4Zf+F74V/wDllQAf8Lr+Df8A0Vr4Zf8Ahe+Ff/llQAf8Lr+Df/RWvhl/4XvhX/5ZUAH/AAuv4N/9Fa+GX/he+Ff/AJZUAH/C6/g3/wBFa+GX/he+Ff8A5ZUAH/C6/g3/ANFa+GX/AIXvhX/5ZUAH/C6/g3/0Vr4Zf+F74V/+WVAB/wALr+Df/RWvhl/4XvhX/wCWVAB/wuv4N/8ARWvhl/4XvhX/AOWVAB/wuv4N/wDRWvhl/wCF74V/+WVAB/wuv4N/9Fa+GX/he+Ff/llQAf8AC6/g3/0Vr4Zf+F74V/8AllQB/Nd/wsv4c/8ARQPBP/hVaD/8nUAf/9kAAFBLAwQUAAYACAAAACEAuN5y8JsDAACACQAAEQAAAHdvcmQvc2V0dGluZ3MueG1stFZLj9s2EL4X6H8wdK5Wj8iOV403sL1xs8E6WazcS2+URNnE8iEMKatO0f/eESWunGYRuA3ii8n55s1vxn7z9k/BJ0cKmim58KKr0JtQWaiSyf3C+3238efeRBsiS8KVpAvvRLX39ubnn960qabGoJqeoAupU1EsvIMxdRoEujhQQfSVqqlEsFIgiMEr7ANB4Kmp/UKJmhiWM87MKYjDcOYNbtTCa0CmgwtfsAKUVpXpTFJVVaygw5ezgEvi9ia3qmgElcZGDIByzEFJfWC1dt7E//WG4ME5OX6riKPgTq+NwgvKbRWUzxaXpNcZ1KAKqjU+kOAuQSbHwMlXjp5jX2HsoUTrCs2j0J7OM5/+NwfxvxxofkklPXTPciDQ82QoQxTp3V4qIDlHVmI5E8zIu0FaflZKTNq0plDg2yCnw9ALOgA7oqrMEEMR1jXl3JK84JSgwzbdAxFITyexNiWtSMPNjuSZUTUqHQnm/Tqc9/DhVB+otCT6A8fD4Uk87fHiQIAUhkJWkwKjrZU0oLjTK9VHZdY4CoAv1VtocqQPQI+Mtg+sMA3Q3pGdl/GU9bOHjiQR2IAv5mmrStoV1AC7/I06A5tU5HJ/MZDCXQGspLuu8Zk5cbrBmjL2mS5l+aHRhqFH25DvyOBbCWC7MfInpMruVNMNJV2P9A8KZh9ow1m9ZQAK7mSJlPphwVhVUcAADCm6RdYxUK3t83tKSlzR3xk3OKcRLvxSu8OjUsaphuF8Fs5vN32mHXoJslxGr5fJS8jqOpldW0oFz1FF2i3LB3CnjkIT0VusiciBkcm2W6dBp5HD04pJh+cU9wM9R7Imd6Dv94AWhPMNjp4D7AoQacl0fUsre+ZbAvvR76ABL0pxDXx49tWtFQq/gWrqHm2B1D01nEqUJIMlk+aeCSfXTZ45K4kb7QxqZPnpCLZPY3va1OAT2xG7J5YqVrcCf/M4UIlD1tGAbkld92zK99HC42x/MFFHAIO3En917SXfxwMWWyzuMXshRVcZag+HURY72ZneKyd7NcoSJ0tG2dTJpqNs5mSzToZLlALu4icktjt28kpxrlpavh/xr0RuSxcMXzw7iXxcrr/0GGcaJ63GPWwUOOxXi0VJWqriDsmKp/655+t3yTxa9vDU7m+zQx49YWsfabUimpYD5kynvelfm+4zj1f+MrqN/WQ2XfnzeP3OX23iZbReXs+m6/jvYQ7cX6ebfwAAAP//AwBQSwMEFAAGAAgAAAAhAPC8NQHcAQAA8QUAABIAAAB3b3JkL2ZvbnRUYWJsZS54bWy8k9tq4zAQhu8LfQej+8ay4vRg6pQ0bWBh6cXSfQBFkW2xOhhJiTdvvyPZcQMhbJallUHI/4x+jT40j0+/lUx23DphdImyCUYJ18xshK5L9PN9dXOPEuep3lBpNC/Rnjv0NL++euyKymjvEtivXaFYiRrv2yJNHWu4om5iWq4hWBmrqIdfW6eK2l/b9oYZ1VIv1kIKv08JxrdosLGXuJiqEoy/GLZVXPu4P7VcgqPRrhGtO7h1l7h1xm5aaxh3Du6sZO+nqNCjTZafGCnBrHGm8hO4zFBRtILtGY4rJT8MZv9mQEYDxYpvtTaWriXAh0oSMEPzgX7SFZoqCCypFGsrYqCl2jieQWxHZYkwwSs8gzl8OZ6GGaUhkTXUOh5M+kTcyxVVQu4PKt160+ut8Kw5yDtqRaipDzlRQ2Dr1rhErxgGWa1Qr2QlykFYLEeFhKPiyAZlOio4KCz69BkPcReLPmMOnJn2AE5AvAvFXfLGu+SHUVSfAULwLYCYAY4AZvr5QMji9QjIEpS7+/xw/Q8gD38H0mO8HMgCypJnMDwDhnx4GfF1fD6G43cxYJh+BYahQZLvom782TYJzfFFbbIIFZPjVxHahOC75xMc8fL/2SbDws3/AAAA//8DAFBLAwQUAAYACAAAACEA4IvKVR8BAAARAgAAFAAAAHdvcmQvd2ViU2V0dGluZ3MueG1slNFRS8MwEAfwd8HvUPK+pRs6tKwbgkz2MgbVD5Cl1zWY5EIua7dv71nnRHyZbzku9+P+3Hx5dDbrIJJBX4rJOBcZeI218ftSvL2uRg8io6R8rSx6KMUJSCwXtzfzvuhhV0FK/JMyVjwVTpeiTSkUUpJuwSkaYwDPzQajU4nLuJdOxfdDGGl0QSWzM9akk5zm+UycmXiNgk1jNDyjPjjwaZiXESyL6Kk1gb61/hqtx1iHiBqIOI+zX55Txl+Yyd0fyBkdkbBJYw5z3migeHySDy9nf4D7/wHTC+B0sd57jGpn+QS8ScaYWPANlLXYbzcv8rOocYOpUh08UcUpLKyMhaETzBEsbSGuvW6zvuiULcXjTHBT/jrk4gMAAP//AwBQSwMEFAAGAAgAAAAhABZNBGBtAQAA7wIAABEACAFkb2NQcm9wcy9jb3JlLnhtbCCiBAEooAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJySUW+CMBSF35fsP5C+Q4suxhDAZDM+zcRkLlv21rVX7YS2aavIv18BxbH5tLd7e757uJw2nZ3KIjiCsULJDMURQQFIpriQ2wy9rhfhFAXWUclpoSRkqAaLZvn9Xcp0wpSBlVEajBNgA+8kbcJ0hnbO6QRjy3ZQUht5Qnpxo0xJnW/NFmvK9nQLeETIBJfgKKeO4sYw1L0jOlty1lvqgylaA84wFFCCdBbHUYyvrANT2psDrfKDLIWrNdxEL2JPn6zowaqqomrcon7/GL8vn1/aXw2FbLJigPKUs8QJV0Ce4mvpK3v4/ALmuuO+8TUzQJ0y+ZweBQ9WovBdC12EJvI91JUy3PrxQecxDpYZoZ2/yM58cODpglq39De7EcAf61/f+as3IwaOonkZedwSfZueY+52Ax74eJIuzIvyNn6arxcoH5F4EpJxGJM1mSajh4SQj2a9wfzVsDwv8G/Hi0GX0PCJ5t8AAAD//wMAUEsDBBQABgAIAAAAIQCBlv05MgsAAGRyAAAPAAAAd29yZC9zdHlsZXMueG1svJ3bctu6FYbvO9N34OiqvXB8jJ14trPHduLaUzvbO3Kaa4iEJNQgofLgQ5++IEhJkBdBcQGrvrIlan0A8eMHsEBS+u33l1RGTzwvhMrORvsf9kYRz2KViGx2Nvr5cLXzaRQVJcsSJlXGz0avvBj9/uWvf/nt+bQoXyUvIg3IitM0PhvNy3JxurtbxHOesuKDWvBMH5yqPGWlfpnPdlOWP1aLnVilC1aKiZCifN092Ns7HrWYfAhFTaci5l9VXKU8K038bs6lJqqsmItFsaQ9D6E9qzxZ5CrmRaFPOpUNL2UiW2H2jwAoFXGuCjUtP+iTaWtkUDp8f8/8l8o14CMOcLACpPHpzSxTOZtI3fq6JpGGjb7o5k9U/JVPWSXLon6Z3+fty/aV+XOlsrKInk9ZEQvxoEvWkFRo3vV5VoiRPsJZUZ4XgnUenNf/dB6Ji9J6+0IkYrRbl1j8Vx98YvJsdHC0fOeyrsHGe5Jls+V703zn6oddk7MRz3Z+juu3Jpp7NmL5zvi8DtxtT6z5a53uYvWq+dSbttFdQ3eUcdNf9VE+vVXxI0/GpT5wNtqri9Jv/ry5z4XKdZ88G33+3L455qm4FknCM+uD2Vwk/NecZz8Lnqzf//PK9Kv2jVhVmf7/8NOe0UsWybeXmC/qXqqPZqxuve91gKw/XYl14Sb8P0vYfttmXfFzzmqrRvtvEab6KMRBHVFYZ9vNrN6cu/kUqqDD9yro6L0K+vheBR2/V0En71XQp/cqyGD+nwWJLOEvjRFhMYC6jeNwI5rjMBua4/ASmuOwCprjcAKa4+joaI6jH6M5jm6K4JQqdvVCq7MfOnp7P3f7HOHH3T4l+HG3zwB+3O0Dvh93+/jux90+nPtxt4/eftztgzWe2yy1ohtts6wMdtlUqTJTJY9K/hJOY5lmmfyFhldPejwnOUkCTDOytRNxMC1m5vX2HmJM6j+fl3XKFalpNBWzKtdpb2jFefbEpU5AI5YkmkcIzHlZ5Y4W8enTOZ/ynGcxp+zYdFApMh5lVToh6JsLNiNj8Swhbr4lkWRQWHVoVpXz2iSCoFOnLM5VeNUUIxsfbkUR3lY1JLqopORErO80XcywwnMDgwlPDQwmPDMwmPDEwNKMqolaGlFLtTSiBmtpRO3W9E+qdmtpRO3W0ojaraWFt9uDKKUZ4u1Vx/7wvbtLqeod5+B6jMUsY3oBED7dtHum0T3L2Sxni3lU7x93Y+1zxpZzoZLX6IFiTluRqNb1potc6rMWWRXeoBs0KnOteET2WvGIDLbihVvsTi+T6wXaNU0+M64mZadpDWmQacdMVs2CNtxtrAzvYWsDXIm8ILNBN5agB3+vl7O1nBQj37qW4RVbs8Jt9XZUIq1eiySopVTxI80wfP264LlOyx6DSVdKSvXMEzriuMxV09dsyx8YSQZZ/lu6mLNCmFxpAzF8ql9eq47u2CL4hO4lExmNbt92UiZkRLeCuH64u40e1KJOM+uGoQFeqLJUKRmz3Qn82y8++TtNBc91Epy9Ep3tOdH2kIFdCoJJpiGphIikl5kiEyRzqOH9k79OFMsTGtp9zpvbQ0pORByzdNEsOgi8pcfFZz3+EKyGDO9fLBf1vhCVqR5IYNa2YVFN/s3j8KHuu4pIdob+qEqz/2iWuiaaDhe+TNjAhS8RjJp6eqj7L8HJbuDCT3YDR3Wyl5IVhXBeQvXmUZ3ukkd9vuHJX8tTUuXTStI14BJI1oJLIFkTKlmlWUF5xoZHeMKGR32+hF3G8Ai25AzvH7lIyMQwMColDIxKBgOj0sDASAUIv0PHgoXfpmPBwu/VaWBESwALRtXPSKd/oqs8FoyqnxkYVT8zMKp+ZmBU/ezwa8SnU70IpptiLCRVn7OQdBNNVvJ0oXKWvxIhv0k+YwQbpA3tPlfT+rkBlTU3cRMg6z1qSbjYbnBUIv/iE7Kq1SzKehHsiDIplSLaW1tPOCZy8961bWHmmYvgKpjN9lv+xClW4xaM6DJAAwuXzYKFT1MWLHyasmDh05QFC5+mLFj4NGXBwu9fvpcs5nMlE547jNhXkWi8YHF7bQlcox60V38rZvMyGs9Xl6hszPHe1sjlLtNG2PYCuwaK44OesDueiCpdVhQ+AXR8ODzYGHojePmgVk/wevm7EflxYCQs83h75Dq124g8GRgJy/w0MNKMUhuRfYP4V5Y/dnaEk77+s9qYcHS+k75etAruLLavI60iu7rgSV8v2rBKdB7H9SUuqM4wz7jjh5nHHY9xkZuCsZObMthXbkSfwX7wJ1EvRzGDpilvdcvP2+IOzZQ6aOT8s1LNxaaNq6TDn0S80av9rOBRJ+dw+NXWjVHG3Y6Dhxs3YvC440YMHoDciEEjkTMcNSS5KYPHJjdi8CDlRqBHKzgj4EYrGI8brWC8z2gFKT6jVcAqwI0YvBxwI9BGhQi0UQNWCm4Eyqgg3MuokII2KkSgjQoRaKPCBRjOqDAeZ1QY72NUSPExKqSgjQoRaKNCBNqoEIE2KkSgjeq5tneGexkVUtBGhQi0USECbVSzXgwwKozHGRXG+xgVUnyMCiloo0IE2qgQgTYqRKCNChFoo0IEyqgg3MuokII2KkSgjQoRaKM2z8f6GxXG44wK432MCik+RoUUtFEhAm1UiEAbFSLQRoUItFEhAmVUEO5lVEhBGxUi0EaFCLRRzaWDAKPCeJxRYbyPUSHFx6iQgjYqRKCNChFoo0IE2qgQgTYqRKCMCsK9jAopaKNCBNqoENHXP9vr6q5nQ/bxu57Ox0yGX7pqK/XD/v4BG3U4HLWslZs1/AGaC6Ueo86nZQ9NvjEMIiZSKLNF7bgXxOaaC6Soq/V/XPY/lmbTA78prH2Ax1zoB/CjoZFgT+Wor8vbkSDJO+rr6XYkWHUe9Y2+diSYBo/6Bl3jy+WdVHo6AsF9w4wVvO8I7xutrXDYxH1jtBUIW7hvZLYCYQP3jcdW4MeoHpzfRn8c2E7Hq5uiAaGvO1qEEzehr1tCrZbDMTTGUNHchKHquQlDZXQTUHo6MXhh3Si0wm6Un9TQZlip/Y3qJmClhgQvqQHGX2qI8pYaovykhgMjVmpIwErtPzi7CV5SA4y/1BDlLTVE+UkNpzKs1JCAlRoSsFIHTshOjL/UEOUtNUT5SQ0Xd1ipIQErNSRgpYYEL6kBxl9qiPKWGqL8pAZZMlpqSMBKDQlYqSHBS2qA8Zcaorylhqg+qc0uyobUKIWtcNwizArETchWIG5wtgI9siUr2jNbsgie2RLUaqk5LluyRXMThqrnJgyV0U1A6enE4IV1o9AKu1F+UuOypS6p/Y3qJmClxmVLTqlx2VKv1LhsqVdqXLbklhqXLXVJjcuWuqT2H5zdBC+pcdlSr9S4bKlXaly25JYaly11SY3LlrqkxmVLXVIHTshOjL/UuGypV2pctuSWGpctdUmNy5a6pMZlS11S47Ilp9S4bKlXaly21Cs1LltyS43LlrqkxmVLXVLjsqUuqXHZklNqXLbUKzUuW+qV2pEt7T5v/GpYzTa/d6c/XL4ueP3F8dYDM0nzxbntRUDzwZtk9etedXBdk6j9xbP2bVPh9oJhU6IJhEXFc11W3H7ll6OoeyWFPm+WJ/pwCYp0fLOvqcL65JefbhtzfRG0+dzGBc/eGpd1Y/fU1ojBqt72aRRzVfFz2wW31VHXaCKbH8PT/9xkiQY8t7+w1tQ1eWENSh+/5FLesebTauH+qOTTsjm6v2cen31zfNJ8YaEzPjeDhBOwu1mZ5mX7w3eOFm9+wqC9eu1o9fMqrjIutRt4R5ub+ylCm3tdweV/xZf/AQAA//8DAFBLAwQUAAYACAAAACEAQP7QLGkBAAC3AgAAEAAIAWRvY1Byb3BzL2FwcC54bWwgogQBKKAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACcUk1LxTAQvAv+h9K7L32CH8i+iCjiQUV4Vc8h2bbBNAnJKr5/78ZqrXgzp92ZZHZmCZy/j656w5Rt8Jt6vWrqCr0Oxvp+Uz+21wendZVJeaNc8Lipd5jrc7m/Bw8pRExkMVcs4fOmHojimRBZDziqvGLaM9OFNCriNvUidJ3VeBX064iexGHTHAt8J/QGzUGcBetJ8eyN/itqgi7+8lO7i6wnocUxOkUo78tLtzKBRhAzCm0g5Vo7omwYnht4UD1muQYxFfAcksnyEMRUwOWgktLE+5PrExCLFi5idFYr4sXKO6tTyKGj6k5p6ynkoSoKIJa3gENsUb8mS7viY9nCrfWTk6lgZ0n1ScXhy97cwVYrh5ccX3bKZQTxAxSVl/wY23BVYn/xv8FFpmdLwzYqXQafLtMtCNgyioa9zuNmAG54/ckVeX7rezTfd/4SZV9P0z+U66NVw+dzO98YZ5w/iPwAAAD//wMAUEsBAi0AFAAGAAgAAAAhAG2KJ0tmAQAAVAUAABMAAAAAAAAAAAAAAAAAAAAAAFtDb250ZW50X1R5cGVzXS54bWxQSwECLQAUAAYACAAAACEAx8InvP8AAADfAgAACwAAAAAAAAAAAAAAAACfAwAAX3JlbHMvLnJlbHNQSwECLQAUAAYACAAAACEAE6o+h/YAAAAxAwAAHAAAAAAAAAAAAAAAAADPBgAAd29yZC9fcmVscy9kb2N1bWVudC54bWwucmVsc1BLAQItABQABgAIAAAAIQD1Yo5gZQIAAA4HAAARAAAAAAAAAAAAAAAAAAcJAAB3b3JkL2RvY3VtZW50LnhtbFBLAQItABQABgAIAAAAIQBtTVmrIQYAAI4aAAAVAAAAAAAAAAAAAAAAAJsLAAB3b3JkL3RoZW1lL3RoZW1lMS54bWxQSwECLQAKAAAAAAAAACEAvOgH/fQnAAD0JwAAFwAAAAAAAAAAAAAAAADvEQAAZG9jUHJvcHMvdGh1bWJuYWlsLmpwZWdQSwECLQAUAAYACAAAACEAuN5y8JsDAACACQAAEQAAAAAAAAAAAAAAAAAYOgAAd29yZC9zZXR0aW5ncy54bWxQSwECLQAUAAYACAAAACEA8Lw1AdwBAADxBQAAEgAAAAAAAAAAAAAAAADiPQAAd29yZC9mb250VGFibGUueG1sUEsBAi0AFAAGAAgAAAAhAOCLylUfAQAAEQIAABQAAAAAAAAAAAAAAAAA7j8AAHdvcmQvd2ViU2V0dGluZ3MueG1sUEsBAi0AFAAGAAgAAAAhABZNBGBtAQAA7wIAABEAAAAAAAAAAAAAAAAAP0EAAGRvY1Byb3BzL2NvcmUueG1sUEsBAi0AFAAGAAgAAAAhAIGW/TkyCwAAZHIAAA8AAAAAAAAAAAAAAAAA40MAAHdvcmQvc3R5bGVzLnhtbFBLAQItABQABgAIAAAAIQBA/tAsaQEAALcCAAAQAAAAAAAAAAAAAAAAAEJPAABkb2NQcm9wcy9hcHAueG1sUEsFBgAAAAAMAAwABgMAAOFRAAAAAA==" } @@ -67,7 +64,6 @@ - do: get: index: test - type: test id: 1 - length: { _source.attachment: 6 } - match: { _source.attachment.content: "Test elasticsearch" } @@ -76,4 +72,3 @@ - match: { _source.attachment.date: "2016-03-10T08:24:00Z" } - match: { _source.attachment.content_length: 19 } - match: { _source.attachment.content_type: "application/vnd.openxmlformats-officedocument.wordprocessingml.document" } - diff --git a/plugins/mapper-annotated-text/src/test/resources/rest-api-spec/test/mapper_annotatedtext/10_basic.yml b/plugins/mapper-annotated-text/src/test/resources/rest-api-spec/test/mapper_annotatedtext/10_basic.yml index 461e9a7ec5040..0049e60def439 100644 --- a/plugins/mapper-annotated-text/src/test/resources/rest-api-spec/test/mapper_annotatedtext/10_basic.yml +++ b/plugins/mapper-annotated-text/src/test/resources/rest-api-spec/test/mapper_annotatedtext/10_basic.yml @@ -9,24 +9,23 @@ - do: indices.create: + include_type_name: false index: annotated body: settings: number_of_shards: "1" number_of_replicas: "0" mappings: - doc: - properties: - text: - type: annotated_text - entityID: - type: keyword + properties: + text: + type: annotated_text + entityID: + type: keyword - do: index: index: annotated - type: doc - body: + body: "text" : "The [quick brown fox](entity_3789) is brown." "entityID": "entity_3789" refresh: true diff --git a/plugins/mapper-murmur3/src/test/resources/rest-api-spec/test/mapper_murmur3/10_basic.yml b/plugins/mapper-murmur3/src/test/resources/rest-api-spec/test/mapper_murmur3/10_basic.yml index d24c3d1eec21c..647fd52f17ef3 100644 --- a/plugins/mapper-murmur3/src/test/resources/rest-api-spec/test/mapper_murmur3/10_basic.yml +++ b/plugins/mapper-murmur3/src/test/resources/rest-api-spec/test/mapper_murmur3/10_basic.yml @@ -6,15 +6,15 @@ - do: indices.create: + include_type_name: false index: test body: mappings: - type1: { "properties": { "foo": { "type": "text", "fields": { "hash": { "type": "murmur3" } } } } } + properties: { "foo": { "type": "text", "fields": { "hash": { "type": "murmur3" } } } } - do: index: index: test - type: type1 id: 0 body: { "foo": null } @@ -31,28 +31,24 @@ - do: index: index: test - type: type1 id: 1 body: { "foo": "bar" } - do: index: index: test - type: type1 id: 2 body: { "foo": "baz" } - do: index: index: test - type: type1 id: 3 body: { "foo": "quux" } - do: index: index: test - type: type1 id: 4 body: { "foo": "bar" } diff --git a/plugins/mapper-size/src/test/resources/rest-api-spec/test/mapper_size/10_basic.yml b/plugins/mapper-size/src/test/resources/rest-api-spec/test/mapper_size/10_basic.yml index be08c46034ad5..9a1049f72d0a4 100644 --- a/plugins/mapper-size/src/test/resources/rest-api-spec/test/mapper_size/10_basic.yml +++ b/plugins/mapper-size/src/test/resources/rest-api-spec/test/mapper_size/10_basic.yml @@ -6,15 +6,15 @@ - do: indices.create: + include_type_name: false index: test body: mappings: - type1: { "_size": { "enabled": true } } + _size: { "enabled": true } - do: index: index: test - type: type1 id: 1 body: { "foo": "bar" } @@ -24,9 +24,7 @@ - do: get: index: test - type: type1 id: 1 stored_fields: "_size" - gt: { _size: 0 } - diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml index 97bb36163b11d..e6c94f8c408d9 100644 --- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml +++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml @@ -69,17 +69,14 @@ setup: body: - index: _index: docs - _type: doc _id: 1 - snapshot: one - index: _index: docs - _type: doc _id: 2 - snapshot: one - index: _index: docs - _type: doc _id: 3 - snapshot: one @@ -181,22 +178,18 @@ setup: body: - index: _index: docs - _type: doc _id: 4 - snapshot: two - index: _index: docs - _type: doc _id: 5 - snapshot: two - index: _index: docs - _type: doc _id: 6 - snapshot: two - index: _index: docs - _type: doc _id: 7 - snapshot: two diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml index 97ed9b3886945..d5bdcd9c4f203 100644 --- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml +++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml @@ -40,17 +40,14 @@ setup: body: - index: _index: docs - _type: doc _id: 1 - snapshot: one - index: _index: docs - _type: doc _id: 2 - snapshot: one - index: _index: docs - _type: doc _id: 3 - snapshot: one @@ -88,22 +85,18 @@ setup: body: - index: _index: docs - _type: doc _id: 4 - snapshot: two - index: _index: docs - _type: doc _id: 5 - snapshot: two - index: _index: docs - _type: doc _id: 6 - snapshot: two - index: _index: docs - _type: doc _id: 7 - snapshot: two diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml index 9af7b5710a360..829ae6197659c 100644 --- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml +++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml @@ -40,17 +40,14 @@ setup: body: - index: _index: docs - _type: doc _id: 1 - snapshot: one - index: _index: docs - _type: doc _id: 2 - snapshot: one - index: _index: docs - _type: doc _id: 3 - snapshot: one @@ -88,22 +85,18 @@ setup: body: - index: _index: docs - _type: doc _id: 4 - snapshot: two - index: _index: docs - _type: doc _id: 5 - snapshot: two - index: _index: docs - _type: doc _id: 6 - snapshot: two - index: _index: docs - _type: doc _id: 7 - snapshot: two diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml index d588a45898ef7..c59d3a32badc7 100644 --- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml +++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml @@ -40,17 +40,14 @@ setup: body: - index: _index: docs - _type: doc _id: 1 - snapshot: one - index: _index: docs - _type: doc _id: 2 - snapshot: one - index: _index: docs - _type: doc _id: 3 - snapshot: one @@ -88,22 +85,18 @@ setup: body: - index: _index: docs - _type: doc _id: 4 - snapshot: two - index: _index: docs - _type: doc _id: 5 - snapshot: two - index: _index: docs - _type: doc _id: 6 - snapshot: two - index: _index: docs - _type: doc _id: 7 - snapshot: two diff --git a/plugins/store-smb/src/test/resources/rest-api-spec/test/store_smb/15_index_creation.yml b/plugins/store-smb/src/test/resources/rest-api-spec/test/store_smb/15_index_creation.yml index 53b036b6682b5..09e59c7fc9d9a 100644 --- a/plugins/store-smb/src/test/resources/rest-api-spec/test/store_smb/15_index_creation.yml +++ b/plugins/store-smb/src/test/resources/rest-api-spec/test/store_smb/15_index_creation.yml @@ -10,18 +10,16 @@ - do: index: index: smb-test - type: doc id: 1 body: { foo: bar } - do: get: index: smb-test - type: doc id: 1 - match: { _index: smb-test } - - match: { _type: doc } + - match: { _type: _doc } - match: { _id: "1"} - match: { _version: 1} - match: { _source: { foo: bar }} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yml index 5892077236ca3..0c64c94ce7f85 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yml @@ -54,13 +54,13 @@ - do: indices.create: + include_type_name: false index: test body: mappings: - type_1: - properties: - foo: - type: text + properties: + foo: + type: text - do: indices.put_alias: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.count/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.count/10_basic.yml index 87ca75a609264..7a6a29032cf74 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.count/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.count/10_basic.yml @@ -24,7 +24,6 @@ - do: index: index: index1 - type: type1 id: 1 body: { foo: bar } refresh: true @@ -40,7 +39,6 @@ - do: index: index: index2 - type: type2 id: 1 body: { foo: bar } refresh: true diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.fielddata/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.fielddata/10_basic.yml index 6c57a31de1488..b1b0e5fac933d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.fielddata/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.fielddata/10_basic.yml @@ -22,21 +22,20 @@ - do: indices.create: + include_type_name: false index: index body: settings: number_of_shards: "1" mappings: - type: - properties: - foo: - type: text - fielddata: true + properties: + foo: + type: text + fielddata: true - do: index: index: index - type: type body: { foo: bar } refresh: true diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yml index 241ff710568fe..69ceccc1ef3bf 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yml @@ -11,7 +11,6 @@ - do: index: index: index1 - type: type1 id: 1 body: { foo: bar } refresh: true diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.segments/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.segments/10_basic.yml index ba2684dc4124c..82488a0c23ce2 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.segments/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.segments/10_basic.yml @@ -42,7 +42,6 @@ - do: index: index: index1 - type: type body: { foo: bar } refresh: true - do: @@ -62,7 +61,6 @@ - do: index: index: index2 - type: type body: { foo: bar } refresh: true - do: @@ -117,7 +115,6 @@ - do: index: index: foo - type: type body: { test: foo } refresh: true @@ -132,7 +129,6 @@ - do: index: index: bar - type: type body: { test: bar } refresh: true @@ -147,7 +143,6 @@ - do: index: index: baz - type: type body: { test: baz } refresh: true diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml index 16551ede70b15..adb459860ef35 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml @@ -235,7 +235,6 @@ - do: index: index: bar - type: type body: { test: bar } refresh: true diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.state/20_filtering.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.state/20_filtering.yml index 861e1200991b1..88da42ee876be 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.state/20_filtering.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.state/20_filtering.yml @@ -2,7 +2,6 @@ setup: - do: index: index: testidx - type: testtype id: testing_document body: "text" : "The quick brown fox is brown." @@ -98,7 +97,6 @@ setup: - do: index: index: another - type: type id: testing_document body: "text" : "The quick brown fox is brown." @@ -131,7 +129,6 @@ setup: - do: index: index: index1 - type: type id: testing_document body: "text" : "The quick brown fox is brown." @@ -139,7 +136,6 @@ setup: - do: index: index: index2 - type: type id: testing_document body: "text" : "The quick brown fox is brown." diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/count/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/count/10_basic.yml index a8ef8c01e51a9..09d96670f688e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/count/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/count/10_basic.yml @@ -5,7 +5,6 @@ setup: - do: index: index: test - type: test id: 1 body: { foo: bar } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/count/20_query_string.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/count/20_query_string.yml index bf644d9a5f1c5..5ef6b7584452a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/count/20_query_string.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/count/20_query_string.yml @@ -2,18 +2,17 @@ "count with query_string parameters": - do: indices.create: + include_type_name: false index: test body: mappings: - test: - properties: - number: - type: integer + properties: + number: + type: integer - do: index: index: test - type: test id: 1 body: { field: foo bar} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/exists/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/exists/10_basic.yml index d98cbfca5b4ad..1ab90e3efa83f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/exists/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/exists/10_basic.yml @@ -14,7 +14,6 @@ - do: index: index: test_1 - type: _doc id: 1 body: { "foo": "bar" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/exists/40_routing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/exists/40_routing.yml index a4ff03517293c..8d59c8a0535f5 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/exists/40_routing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/exists/40_routing.yml @@ -21,7 +21,6 @@ - do: index: index: test_1 - type: _doc id: 1 routing: 5 body: { foo: bar } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/exists/60_realtime_refresh.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/exists/60_realtime_refresh.yml index 2316831922732..e12a504349c4d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/exists/60_realtime_refresh.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/exists/60_realtime_refresh.yml @@ -20,7 +20,6 @@ - do: index: index: test_1 - type: _doc id: 1 body: { foo: bar } @@ -31,7 +30,7 @@ realtime: false - is_false: '' - + - do: exists: index: test_1 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/exists/70_defaults.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/exists/70_defaults.yml index c1e113037678b..6fabdd59820cf 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/exists/70_defaults.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/exists/70_defaults.yml @@ -7,7 +7,6 @@ - do: index: index: test_1 - type: _doc id: 1 body: { "foo": "bar" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/explain/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/explain/10_basic.yml index be40e6357e865..bfe8da8d91519 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/explain/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/explain/10_basic.yml @@ -14,7 +14,6 @@ setup: - do: index: index: test_1 - type: _doc id: id_1 body: { foo: bar, title: howdy } @@ -64,4 +63,3 @@ setup: id: id_1 body: match_all: {} - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/explain/20_source_filtering.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/explain/20_source_filtering.yml index 2302dcf3576be..ad596f980807b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/explain/20_source_filtering.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/explain/20_source_filtering.yml @@ -7,7 +7,6 @@ - do: index: index: test_1 - type: _doc id: 1 body: { "include": { "field1": "v1", "field2": "v2" }, "count": 1 } - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/explain/30_query_string.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/explain/30_query_string.yml index 9415c3f3e7a3a..6a71cba7c4da1 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/explain/30_query_string.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/explain/30_query_string.yml @@ -6,18 +6,17 @@ - do: indices.create: + include_type_name: false index: test body: mappings: - _doc: - properties: - number: - type: integer + properties: + number: + type: integer - do: index: index: test - type: _doc id: 1 body: { field: foo bar} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/10_basic.yml index 2c7937aeaccb7..7fe06f1c56152 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/10_basic.yml @@ -2,104 +2,104 @@ setup: - do: indices.create: + include_type_name: false index: test1 body: mappings: - t: - properties: - text: - type: text - keyword: - type: keyword - number: - type: double - geo: - type: geo_point - object: - type: object - properties: - nested1 : - type : text - index: false - nested2: - type: float - doc_values: false - level1: - type: nested - properties: - level2: - type: object - properties: - leaf1: - type: text - index: false + properties: + text: + type: text + keyword: + type: keyword + number: + type: double + geo: + type: geo_point + object: + type: object + properties: + nested1 : + type : text + index: false + nested2: + type: float + doc_values: false + level1: + type: nested + properties: + level2: + type: object + properties: + leaf1: + type: text + index: false - do: indices.create: + include_type_name: false index: test2 body: mappings: - t: - properties: - text: - type: text - keyword: - type: keyword - number: - type: double - geo: - type: geo_point - object: - type: object - properties: - nested1 : - type : text - index: true - nested2: - type: float - doc_values: true - level1: - type: nested - properties: - level2: - type: object - properties: - leaf1: - type: text - index: false + properties: + text: + type: text + keyword: + type: keyword + number: + type: double + geo: + type: geo_point + object: + type: object + properties: + nested1 : + type : text + index: true + nested2: + type: float + doc_values: true + level1: + type: nested + properties: + level2: + type: object + properties: + leaf1: + type: text + index: false - do: indices.create: + include_type_name: false index: test3 body: mappings: - t: - properties: - text: - type: text - keyword: - type: keyword - number: - type: long - geo: - type: keyword - object: - type: nested - properties: - nested1 : - type : long - index: false - nested2: - type: keyword - doc_values: false - level1: - type: object - properties: - level2: - type: object - properties: - leaf1: - type: text - index: false + properties: + text: + type: text + keyword: + type: keyword + number: + type: long + geo: + type: keyword + object: + type: nested + properties: + nested1 : + type : long + index: false + nested2: + type: keyword + doc_values: false + level1: + type: object + properties: + level2: + type: object + properties: + leaf1: + type: text + index: false --- "Get simple field caps": diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/15_default_values.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/15_default_values.yml index a3739823766f1..57c11a1ca10e2 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/15_default_values.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/15_default_values.yml @@ -9,7 +9,6 @@ - do: index: index: test_1 - type: test id: 1 body: { "foo": "bar" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/40_routing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/40_routing.yml index 7a4cb421f2eac..6425f70f26aad 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/40_routing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/40_routing.yml @@ -23,7 +23,6 @@ - do: index: index: test_1 - type: test id: 1 routing: 5 body: { foo: bar } @@ -41,4 +40,3 @@ get_source: index: test_1 id: 1 - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/60_realtime_refresh.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/60_realtime_refresh.yml index 6afaafe5da434..d39b07a6ce5f7 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/60_realtime_refresh.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/60_realtime_refresh.yml @@ -20,7 +20,6 @@ - do: index: index: test_1 - type: test id: 1 body: { foo: bar } @@ -47,4 +46,3 @@ refresh: true - match: { '': {foo: bar}} - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/70_source_filtering.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/70_source_filtering.yml index 3687ec40f0bb9..2665458cea95d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/70_source_filtering.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/70_source_filtering.yml @@ -9,7 +9,6 @@ - do: index: index: test_1 - type: test id: 1 body: { "include": { "field1": "v1", "field2": "v2" }, "count": 1 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/85_source_missing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/85_source_missing.yml index 78f2760a04944..79b49097e743a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/85_source_missing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/85_source_missing.yml @@ -7,16 +7,15 @@ setup: - do: indices.create: + include_type_name: false index: test_1 body: mappings: - test: - _source: { enabled: false } + _source: { enabled: false } - do: index: index: test_1 - type: test id: 1 body: { foo: bar } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/10_analyze.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/10_analyze.yml index 85861a23d5943..0b1c205e8956c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/10_analyze.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/10_analyze.yml @@ -11,14 +11,14 @@ "Index and field": - do: indices.create: + include_type_name: false index: test body: mappings: - test: - properties: - text: - type: text - analyzer: standard + properties: + text: + type: text + analyzer: standard - do: indices.analyze: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/10_basic.yml index e7c0af2ca1422..6f5ae97683b82 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/10_basic.yml @@ -3,13 +3,16 @@ setup: - do: indices.create: + include_type_name: false index: test_index body: aliases: test_alias: {} test_blias: {} mappings: - type_1: {} + properties: + foo: + type: keyword settings: number_of_shards: 1 number_of_replicas: 1 @@ -52,41 +55,6 @@ setup: - is_true: test_index.settings - is_true: test_index.mappings ---- -"Test include_type_name": - - skip: - version: " - 6.6.99" - reason: the include_type_name parameter is not supported before 6.7 - - - do: - indices.get: - include_type_name: true - index: test_index - - - is_true: test_index.mappings - - is_true: test_index.mappings.type_1 - - - do: - indices.get: - include_type_name: false - index: test_index - - - is_true: test_index.mappings - - is_false: test_index.mappings.type_1 - ---- -"Test include_type_name dafaults to false": - - skip: - version: " - 6.99.99" - reason: the include_type_name parameter default is different on 6.x and 7.0, so only test this on 7.0 clusters - - - do: - indices.get: - index: test_index - - - is_true: test_index.mappings - - is_false: test_index.mappings.type_1 - --- "Get index infos should work for wildcards": @@ -198,4 +166,3 @@ setup: catch: bad_request indices.get: index: _foo - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/11_basic_with_types.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/11_basic_with_types.yml new file mode 100644 index 0000000000000..2f3e538743c56 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/11_basic_with_types.yml @@ -0,0 +1,77 @@ +--- +setup: + + - do: + indices.create: + index: test_index + body: + aliases: + test_alias: {} + test_blias: {} + mappings: + type_1: {} + settings: + number_of_shards: 1 + number_of_replicas: 1 + + - do: + indices.create: + index: test_index_2 + body: + settings: + number_of_shards: 1 + number_of_replicas: 2 + aliases: + test_alias: {} + test_blias: {} + + - do: + indices.create: + index: test_index_3 + body: + aliases: + test_alias: {} + test_blias: {} + + - do: + indices.close: + index: test_index_3 + + - do: + cluster.health: + wait_for_status: yellow + +--- +"Test include_type_name": + - skip: + version: " - 6.6.99" + reason: the include_type_name parameter is not supported before 6.7 + + - do: + indices.get: + include_type_name: true + index: test_index + + - is_true: test_index.mappings + - is_true: test_index.mappings.type_1 + + - do: + indices.get: + include_type_name: false + index: test_index + + - is_true: test_index.mappings + - is_false: test_index.mappings.type_1 + +--- +"Test include_type_name dafaults to false": + - skip: + version: " - 6.99.99" + reason: the include_type_name parameter default is different on 6.x and 7.0, so only test this on 7.0 clusters + + - do: + indices.get: + index: test_index + + - is_true: test_index.mappings + - is_false: test_index.mappings.type_1 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_field_mapping/30_missing_type.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_field_mapping/30_missing_type.yml index bbafb3e43f1a7..0bf3f1f7823ee 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_field_mapping/30_missing_type.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_field_mapping/30_missing_type.yml @@ -3,6 +3,7 @@ - do: indices.create: + include_type_name: true index: test_index body: mappings: @@ -19,4 +20,3 @@ index: test_index type: not_test_type fields: text - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/20_missing_type.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/20_missing_type.yml index 2f03bf7df5014..93598cab6cf8b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/20_missing_type.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/20_missing_type.yml @@ -2,6 +2,7 @@ "Non-existent type returns 404": - do: indices.create: + include_type_name: true index: test_index body: mappings: @@ -100,4 +101,3 @@ indices.get_mapping: include_type_name: true type: not_test_type - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/40_aliases.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/40_aliases.yml index 7afd10e1109eb..26086fe4c3a13 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/40_aliases.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/40_aliases.yml @@ -3,10 +3,10 @@ - do: indices.create: + include_type_name: false index: test_index body: mappings: - test_type: properties: text: type: text @@ -24,4 +24,3 @@ - match: {test_index.mappings.properties.text.type: text} - match: {test_index.mappings.properties.text.analyzer: whitespace} - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/50_wildcard_expansion.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/50_wildcard_expansion.yml index 8f8ff8f9fc102..ea6a3d3a01361 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/50_wildcard_expansion.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/50_wildcard_expansion.yml @@ -2,40 +2,52 @@ setup: - do: indices.create: + include_type_name: false index: test-xxx body: settings: index: number_of_replicas: 0 mappings: - type_1: {} + properties: + foo: + type: keyword - do: indices.create: + include_type_name: false index: test-xxy body: settings: index: number_of_replicas: 0 mappings: - type_2: {} + properties: + foo2: + type: keyword - do: indices.create: + include_type_name: false index: test-xyy body: settings: index: number_of_replicas: 0 mappings: - type_3: {} + properties: + foo3: + type: keyword - do: indices.create: + include_type_name: false index: test-yyy body: settings: index: number_of_replicas: 0 mappings: - type_4: {} + properties: + foo4: + type: keyword - do: cluster.health: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/10_basic.yml index cdf64b07f9110..14cc3615db05a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/10_basic.yml @@ -25,7 +25,7 @@ - do: indices.get_mapping: index: test_index - + - match: {test_index.mappings.properties.text1.type: text} - match: {test_index.mappings.properties.text1.analyzer: whitespace} - match: {test_index.mappings.properties.text2.type: text} @@ -47,7 +47,7 @@ - do: indices.get_mapping: index: test_index - + - match: {test_index.mappings.properties.text1.type: text} - match: {test_index.mappings.properties.subfield.properties.text3.type: text} - match: {test_index.mappings.properties.text1.fields.text_raw.type: keyword} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yml index bfd3fc58fdf4b..a6d6bb0730548 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yml @@ -29,18 +29,16 @@ - do: index: index: source - type: test id: "1" body: { "foo": "hello world" } - do: get: index: source - type: test id: "1" - match: { _index: source } - - match: { _type: test } + - match: { _type: _doc } - match: { _id: "1" } - match: { _source: { foo: "hello world" } } @@ -75,10 +73,9 @@ - do: get: index: target - type: test id: "1" - match: { _index: target } - - match: { _type: test } + - match: { _type: _doc } - match: { _id: "1" } - match: { _source: { foo: "hello world" } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/20_source_mapping.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/20_source_mapping.yml index c5715f7d4b2d5..de9c51a890c9a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/20_source_mapping.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/20_source_mapping.yml @@ -14,6 +14,7 @@ # create index - do: indices.create: + include_type_name: false index: source wait_for_active_shards: 1 body: @@ -23,16 +24,14 @@ index.number_of_shards: 2 index.number_of_replicas: 0 mappings: - test: - properties: - count: - type: text + properties: + count: + type: text # index document - do: index: index: source - type: test id: "1" body: { "count": "1" } @@ -75,5 +74,3 @@ - do: cluster.health: wait_for_status: green - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.sort/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.sort/10_basic.yml index b7df22647eb5f..8cc12d4fe959b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.sort/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.sort/10_basic.yml @@ -3,6 +3,7 @@ - do: indices.create: + include_type_name: false index: test body: settings: @@ -10,36 +11,31 @@ number_of_replicas: 0 index.sort.field: rank mappings: - test: - properties: - rank: - type: integer + properties: + rank: + type: integer - do: index: index: test - type: test id: "1" body: { "rank": 4 } - do: index: index: test - type: test id: "2" body: { "rank": 1 } - do: index: index: test - type: test id: "3" body: { "rank": 3 } - do: index: index: test - type: test id: "4" body: { "rank": 2 } @@ -50,35 +46,30 @@ - do: index: index: test - type: test id: "5" body: { "rank": 8 } - do: index: index: test - type: test id: "6" body: { "rank": 6 } - do: index: index: test - type: test id: "7" body: { "rank": 5 } - do: index: index: test - type: test id: "8" body: { "rank": 7 } - do: index: index: test - type: test id: "8" body: { "rank": 7 } @@ -166,4 +157,3 @@ query: {"range": { "rank": { "from": 0 } } } track_total_hits: false size: 3 - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml index 74774f13e212e..2baa82ea78842 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml @@ -12,21 +12,18 @@ setup: - do: index: index: source - type: doc id: "1" body: { "foo": "hello world" } - do: index: index: source - type: doc id: "2" body: { "foo": "hello world 2" } - do: index: index: source - type: doc id: "3" body: { "foo": "hello world 3" } @@ -69,11 +66,10 @@ setup: - do: get: index: target - type: doc id: "1" - match: { _index: target } - - match: { _type: doc } + - match: { _type: _doc } - match: { _id: "1" } - match: { _source: { foo: "hello world" } } @@ -81,11 +77,10 @@ setup: - do: get: index: target - type: doc id: "2" - match: { _index: target } - - match: { _type: doc } + - match: { _type: _doc } - match: { _id: "2" } - match: { _source: { foo: "hello world 2" } } @@ -93,11 +88,10 @@ setup: - do: get: index: target - type: doc id: "3" - match: { _index: target } - - match: { _type: doc } + - match: { _type: _doc } - match: { _id: "3" } - match: { _source: { foo: "hello world 3" } } @@ -118,21 +112,18 @@ setup: - do: index: index: source_one_shard - type: doc id: "1" body: { "foo": "hello world" } - do: index: index: source_one_shard - type: doc id: "2" body: { "foo": "hello world 2" } - do: index: index: source_one_shard - type: doc id: "3" body: { "foo": "hello world 3" } @@ -168,11 +159,10 @@ setup: - do: get: index: target - type: doc id: "1" - match: { _index: target } - - match: { _type: doc } + - match: { _type: _doc } - match: { _id: "1" } - match: { _source: { foo: "hello world" } } @@ -180,11 +170,10 @@ setup: - do: get: index: target - type: doc id: "2" - match: { _index: target } - - match: { _type: doc } + - match: { _type: _doc } - match: { _id: "2" } - match: { _source: { foo: "hello world 2" } } @@ -192,11 +181,10 @@ setup: - do: get: index: target - type: doc id: "3" - match: { _index: target } - - match: { _type: doc } + - match: { _type: _doc } - match: { _id: "3" } - match: { _source: { foo: "hello world 3" } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/20_source_mapping.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/20_source_mapping.yml index 0827860d88fdf..442fe7c896173 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/20_source_mapping.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/20_source_mapping.yml @@ -8,6 +8,7 @@ # create index - do: indices.create: + include_type_name: false index: source wait_for_active_shards: 1 body: @@ -16,16 +17,14 @@ number_of_replicas: 0 index.number_of_routing_shards: 2 mappings: - test: - properties: - count: - type: text + properties: + count: + type: text # index document - do: index: index: source - type: test id: "1" body: { "count": "1" } @@ -69,5 +68,3 @@ - do: cluster.health: wait_for_status: green - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/10_index.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/10_index.yml index 564a482727fa7..1a650ee88eae6 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/10_index.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/10_index.yml @@ -18,14 +18,12 @@ setup: - do: index: index: test1 - type: bar id: 1 body: { "foo": "bar" } - do: index: index: test2 - type: baz id: 1 body: { "foo": "baz" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/11_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/11_metric.yml index 0f373b7177c41..79790935beef2 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/11_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/11_metric.yml @@ -4,14 +4,12 @@ setup: - do: index: index: test1 - type: bar id: 1 body: { "foo": "bar" } - do: index: index: test2 - type: baz id: 1 body: { "foo": "baz" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/12_level.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/12_level.yml index c766f5eb62508..e9bd219a3e0ab 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/12_level.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/12_level.yml @@ -4,14 +4,12 @@ setup: - do: index: index: test1 - type: bar id: 1 body: { "foo": "bar" } - do: index: index: test2 - type: baz id: 1 body: { "foo": "baz" } @@ -68,4 +66,3 @@ setup: - is_true: indices.test2.shards - is_true: indices.test1.shards.0.0.commit.id - is_true: indices.test2.shards.0.0.commit.id - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/13_fields.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/13_fields.yml index cb67ef26b82b7..5e7e266394375 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/13_fields.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/13_fields.yml @@ -3,6 +3,7 @@ setup: - do: indices.create: + include_type_name: false index: test1 wait_for_active_shards: all body: @@ -13,20 +14,19 @@ setup: index.number_of_shards: 3 index.number_of_replicas: 0 mappings: - bar: - properties: - bar: - type: text - fielddata: true - fields: - completion: - type: completion - baz: - type: text - fielddata: true - fields: - completion: - type: completion + properties: + bar: + type: text + fielddata: true + fields: + completion: + type: completion + baz: + type: text + fielddata: true + fields: + completion: + type: completion - do: cluster.health: @@ -35,14 +35,12 @@ setup: - do: index: index: test1 - type: bar id: 1 body: { "bar": "bar", "baz": "baz" } - do: index: index: test1 - type: bar id: 2 body: { "bar": "foo", "baz": "foo" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/14_groups.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/14_groups.yml index 052e243f58563..daf55b38919b2 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/14_groups.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/14_groups.yml @@ -4,7 +4,6 @@ setup: - do: index: index: test1 - type: bar id: 1 body: { "bar": "bar", "baz": "baz" } @@ -77,4 +76,3 @@ setup: - gt: { _all.total.search.groups.bar.query_total: 0 } - is_false: _all.total.search.groups.baz - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/15_types.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/15_types.yml index 9d18c945c14fa..e2f31c3405707 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/15_types.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/15_types.yml @@ -79,4 +79,3 @@ setup: - match: { _all.primaries.indexing.types.bar.index_total: 1 } - is_false: _all.primaries.indexing.types.baz - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml index 15fb7c33f3c32..586a04f065cde 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml @@ -17,7 +17,6 @@ setup: - do: index: index: test - type: bar id: 1 body: { "foo": "bar" } @@ -73,7 +72,6 @@ setup: - do: index: index: test - type: bar id: 1 body: { "foo": "bar" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.validate_query/20_query_string.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.validate_query/20_query_string.yml index 29ff59fc96af1..8ce0f8c01cb4d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.validate_query/20_query_string.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.validate_query/20_query_string.yml @@ -2,15 +2,15 @@ "validate_query with query_string parameters": - do: indices.create: + include_type_name: false index: test body: mappings: - test: - properties: - field: - type: text - number: - type: integer + properties: + field: + type: text + number: + type: integer - do: indices.validate_query: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/10_basic.yml index 64216f16654df..798d699ae80a0 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/10_basic.yml @@ -10,7 +10,6 @@ - do: index: index: test_1 - type: _doc id: 1 body: { foo: bar } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/12_non_existent_index.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/12_non_existent_index.yml index 4e2bf0c5d619a..a1101a903f896 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/12_non_existent_index.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/12_non_existent_index.yml @@ -7,7 +7,6 @@ - do: index: index: test_1 - type: _doc id: 1 body: { foo: bar } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/13_missing_metadata.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/13_missing_metadata.yml index 6138264d68086..2711bed58dbb1 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/13_missing_metadata.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/13_missing_metadata.yml @@ -7,7 +7,6 @@ - do: index: index: test_1 - type: _doc id: 1 body: { foo: bar } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/15_ids.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/15_ids.yml index 110b00c5f203e..fbdc9b265a95a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/15_ids.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/15_ids.yml @@ -11,14 +11,12 @@ - do: index: index: test_1 - type: _doc id: 1 body: { foo: bar } - do: index: index: test_1 - type: _doc id: 2 body: { foo: baz } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/17_default_index.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/17_default_index.yml index 8fb09c419b1c1..d03f99be39517 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/17_default_index.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/17_default_index.yml @@ -10,7 +10,6 @@ - do: index: index: test_1 - type: _doc id: 1 body: { foo: bar } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/20_stored_fields.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/20_stored_fields.yml index 0c8abee652fab..5fefcee57c76b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/20_stored_fields.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/20_stored_fields.yml @@ -6,22 +6,21 @@ - do: indices.create: + include_type_name: false index: test_1 body: mappings: - _doc: - properties: - foo: - type: keyword - store: true - count: - type: integer - store: true + properties: + foo: + type: keyword + store: true + count: + type: integer + store: true - do: index: index: test_1 - type: _doc id: 1 body: { foo: bar } @@ -115,4 +114,3 @@ - match: { docs.3.fields.foo: [bar] } - match: { docs.3._source: { foo: bar }} - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/40_routing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/40_routing.yml index 7abe5e9df1f9f..df2924f274bdf 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/40_routing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/40_routing.yml @@ -21,7 +21,6 @@ - do: index: index: test_1 - type: _doc id: 1 routing: 5 body: { foo: bar } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/60_realtime_refresh.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/60_realtime_refresh.yml index 150f77ae4e893..3b1bfcdca556c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/60_realtime_refresh.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/60_realtime_refresh.yml @@ -20,7 +20,6 @@ - do: index: index: test_1 - type: _doc id: 1 body: { foo: bar } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/70_source_filtering.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/70_source_filtering.yml index 35e3548ca42ad..3a3086cf3616d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/70_source_filtering.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/70_source_filtering.yml @@ -6,13 +6,11 @@ setup: - do: index: index: test_1 - type: _doc id: 1 body: { "include": { "field1": "v1", "field2": "v2" }, "count": 1 } - do: index: index: test_1 - type: _doc id: 2 body: { "include": { "field1": "v1", "field2": "v2" }, "count": 1 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/80_deprecated.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/80_deprecated.yml index db42c00d416bd..0283455350a80 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/80_deprecated.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/80_deprecated.yml @@ -9,14 +9,12 @@ - do: index: index: test_1 - type: _doc id: 1 body: { foo: bar } - do: index: index: test_1 - type: _doc id: 2 body: { foo: baz } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mlt/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mlt/10_basic.yml index 9ad62f09538d8..2457087ad2de5 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mlt/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mlt/10_basic.yml @@ -2,23 +2,22 @@ "Basic mlt": - do: indices.create: + include_type_name: false index: test_1 body: settings: index: number_of_replicas: 0 mappings: - test: - properties: - foo: - type : "text" - title: - type : "text" + properties: + foo: + type : "text" + title: + type : "text" - do: index: index: test_1 - type: test id: 1 body: { foo: bar, title: howdy } @@ -42,4 +41,3 @@ fields: ["title"] - match: {hits.total: 0} - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mlt/20_docs.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mlt/20_docs.yml index 50eeab8aba467..bb1b25a0dcb40 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mlt/20_docs.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mlt/20_docs.yml @@ -9,21 +9,18 @@ - do: index: index: test_1 - type: test id: 1 body: { foo: bar } - do: index: index: test_1 - type: test id: 2 body: { foo: baz } - do: index: index: test_1 - type: test id: 3 body: { foo: foo } @@ -44,12 +41,12 @@ like: - _index: test_1 - _type: test + _type: _doc doc: foo: bar - _index: test_1 - _type: test + _type: _doc _id: 2 - _id: 3 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mlt/30_unlike.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mlt/30_unlike.yml index 8f06086732b9d..abea4c8fbe57a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mlt/30_unlike.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mlt/30_unlike.yml @@ -9,21 +9,18 @@ - do: index: index: test_1 - type: test id: 1 body: { foo: bar baz selected } - do: index: index: test_1 - type: test id: 2 body: { foo: bar } - do: index: index: test_1 - type: test id: 3 body: { foo: bar baz } @@ -43,11 +40,11 @@ more_like_this: like: _index: test_1 - _type: test + _type: _doc _id: 1 unlike: _index: test_1 - _type: test + _type: _doc _id: 3 include: true min_doc_freq: 0 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yml index f9fe244529f28..5b092c9d15e44 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yml @@ -4,28 +4,24 @@ setup: - do: index: index: index_1 - type: test id: 1 body: { foo: bar } - do: index: index: index_1 - type: test id: 2 body: { foo: baz } - do: index: index: index_1 - type: test id: 3 body: { foo: foo } - do: index: index: index_2 - type: test id: 1 body: { foo: foo } @@ -154,6 +150,3 @@ setup: - index: index_1 - query: match: {foo: foo} - - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/20_typed_keys.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/20_typed_keys.yml index 269282e17cd92..cab5c5ca628c9 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/20_typed_keys.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/20_typed_keys.yml @@ -2,57 +2,57 @@ setup: - do: indices.create: + include_type_name: false index: test-0 body: settings: number_of_replicas: 0 mappings: - user: - properties: - index_start_at: - type: integer - integer: - type: integer - float: - type: float - name: - type: keyword - title: - type: completion + properties: + index_start_at: + type: integer + integer: + type: integer + float: + type: float + name: + type: keyword + title: + type: completion - do: indices.create: + include_type_name: false index: test-1 body: settings: number_of_replicas: 0 mappings: - user: - properties: - index_start_at: - type: integer - integer: - type: integer - float: - type: float - name: - type: keyword - title: - type: completion + properties: + index_start_at: + type: integer + integer: + type: integer + float: + type: float + name: + type: keyword + title: + type: completion - do: bulk: refresh: true body: - - '{"index": {"_index": "test-0", "_type": "user"}}' + - '{"index": {"_index": "test-0"}}' - '{"row": 1, "index_start_at": 56, "integer": 38, "float": 12.5713, "name": "Ruth", "bool": true, "title": "doctor"}' - - '{"index": {"_index": "test-0", "_type": "user"}}' + - '{"index": {"_index": "test-0"}}' - '{"row": 2, "index_start_at": 57, "integer": 42, "float": 15.3393, "name": "Jackie", "bool": false}' - - '{"index": {"_index": "test-1", "_type": "user"}}' + - '{"index": {"_index": "test-1"}}' - '{"row": 3, "index_start_at": 58, "integer": 29, "float": 19.0517, "name": "Stephanie", "bool": true}' - - '{"index": {"_index": "test-1", "_type": "user"}}' + - '{"index": {"_index": "test-1"}}' - '{"row": 4, "index_start_at": 59, "integer": 19, "float": 19.3717, "bool": true, "title": "commandant"}' - - '{"index": {"_index": "test-1", "_type": "user"}}' + - '{"index": {"_index": "test-1"}}' - '{"row": 5, "index_start_at": 60, "integer": 0, "float": 17.3349, "name": "Natalie", "bool": false}' --- diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mtermvectors/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mtermvectors/10_basic.yml index cb61bbed9d707..0cdf4bf9aef5c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mtermvectors/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mtermvectors/10_basic.yml @@ -4,18 +4,17 @@ setup: reason: types are required in requests before 7.0.0 - do: indices.create: + include_type_name: false index: testidx body: mappings: - _doc: - properties: - text: - type : "text" - term_vector : "with_positions_offsets" + properties: + text: + type : "text" + term_vector : "with_positions_offsets" - do: index: index: testidx - type: _doc id: testing_document body: {"text" : "The quick brown fox is brown."} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mtermvectors/20_deprecated.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mtermvectors/20_deprecated.yml index 758c09e79a8cd..633857815c9c0 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mtermvectors/20_deprecated.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mtermvectors/20_deprecated.yml @@ -13,19 +13,18 @@ setup: - do: indices.create: + include_type_name: false index: testidx body: mappings: - _doc: - properties: - text: - type : "text" - term_vector : "with_positions_offsets" + properties: + text: + type : "text" + term_vector : "with_positions_offsets" - do: index: index: testidx - type: _doc id: testing_document body: {"text" : "The quick brown fox is brown."} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/range/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/range/10_basic.yml index eac9c23704b50..a63ad16096472 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/range/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/range/10_basic.yml @@ -1,25 +1,25 @@ setup: - do: indices.create: + include_type_name: false index: test body: settings: number_of_replicas: 0 mappings: - doc: - "properties": - "integer_range": - "type" : "integer_range" - "long_range": - "type" : "long_range" - "float_range": - "type" : "float_range" - "double_range": - "type" : "double_range" - "date_range": - "type" : "date_range" - "ip_range": - "type" : "ip_range" + "properties": + "integer_range": + "type" : "integer_range" + "long_range": + "type" : "long_range" + "float_range": + "type" : "float_range" + "double_range": + "type" : "double_range" + "date_range": + "type" : "date_range" + "ip_range": + "type" : "ip_range" --- "Integer range": @@ -27,28 +27,24 @@ setup: - do: index: index: test - type: doc id: 1 body: { "integer_range" : { "gte": 1, "lte": 5 } } - do: index: index: test - type: doc id: 2 body: { "integer_range" : { "gte": 1, "lte": 3 } } - do: index: index: test - type: doc id: 3 body: { "integer_range" : { "gte": 4, "lte": 5 } } - do: index: index: test - type: doc id: 4 body: { "integer_range" : null } @@ -103,21 +99,18 @@ setup: - do: index: index: test - type: doc id: 1 body: { "long_range" : { "gte": 1, "lte": 5 } } - do: index: index: test - type: doc id: 2 body: { "long_range" : { "gte": 1, "lte": 3 } } - do: index: index: test - type: doc id: 3 body: { "long_range" : { "gte": 4, "lte": 5 } } @@ -166,21 +159,18 @@ setup: - do: index: index: test - type: doc id: 1 body: { "float_range" : { "gte": 1, "lte": 5 } } - do: index: index: test - type: doc id: 2 body: { "float_range" : { "gte": 1, "lte": 3 } } - do: index: index: test - type: doc id: 3 body: { "float_range" : { "gte": 4, "lte": 5 } } @@ -229,21 +219,18 @@ setup: - do: index: index: test - type: doc id: 1 body: { "double_range" : { "gte": 1, "lte": 5 } } - do: index: index: test - type: doc id: 2 body: { "double_range" : { "gte": 1, "lte": 3 } } - do: index: index: test - type: doc id: 3 body: { "double_range" : { "gte": 4, "lte": 5 } } @@ -292,21 +279,18 @@ setup: - do: index: index: test - type: doc id: 1 body: { "ip_range" : { "gte": "192.168.0.1", "lte": "192.168.0.5" } } - do: index: index: test - type: doc id: 2 body: { "ip_range" : { "gte": "192.168.0.1", "lte": "192.168.0.3" } } - do: index: index: test - type: doc id: 3 body: { "ip_range" : { "gte": "192.168.0.4", "lte": "192.168.0.5" } } @@ -355,21 +339,18 @@ setup: - do: index: index: test - type: doc id: 1 body: { "date_range" : { "gte": "2017-09-01", "lte": "2017-09-05" } } - do: index: index: test - type: doc id: 2 body: { "date_range" : { "gte": "2017-09-01", "lte": "2017-09-03" } } - do: index: index: test - type: doc id: 3 body: { "date_range" : { "gte": "2017-09-04", "lte": "2017-09-05" } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/10_basic.yml index 3fec62f60e7b9..aa6d1e9841dd7 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/10_basic.yml @@ -6,14 +6,12 @@ - do: index: index: test_scroll - type: test id: 42 body: { foo: 1 } - do: index: index: test_scroll - type: test id: 43 body: { foo: 2 } @@ -39,7 +37,6 @@ - do: index: index: test_scroll - type: test id: 44 body: { foo: 3 } @@ -81,14 +78,12 @@ - do: index: index: test_scroll - type: test id: 42 body: { foo: 1 } - do: index: index: test_scroll - type: test id: 43 body: { foo: 2 } @@ -114,7 +109,6 @@ - do: index: index: test_scroll - type: test id: 44 body: { foo: 3 } @@ -151,14 +145,12 @@ - do: index: index: test_scroll - type: test id: 42 body: { foo: 1 } - do: index: index: test_scroll - type: test id: 43 body: { foo: 2 } @@ -184,7 +176,6 @@ - do: index: index: test_scroll - type: test id: 44 body: { foo: 3 } @@ -256,14 +247,12 @@ - do: index: index: test_scroll - type: test id: 42 body: { foo: 1 } - do: index: index: test_scroll - type: test id: 43 body: { foo: 2 } @@ -305,14 +294,12 @@ - do: index: index: test_scroll - type: test id: 42 body: { foo: 1 } - do: index: index: test_scroll - type: test id: 43 body: { foo: 2 } @@ -356,4 +343,3 @@ - do: clear_scroll: scroll_id: $scroll_id - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/11_clear.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/11_clear.yml index 40eb1fb004d54..97a13dd0c2c5f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/11_clear.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/11_clear.yml @@ -6,7 +6,6 @@ - do: index: index: test_scroll - type: test id: 42 body: { foo: bar } @@ -33,7 +32,7 @@ scroll: rest_total_hits_as_int: true scroll_id: $scroll_id1 - + - do: catch: missing clear_scroll: @@ -47,7 +46,6 @@ - do: index: index: test_scroll - type: test id: 42 body: { foo: bar } @@ -89,7 +87,6 @@ - do: index: index: test_scroll - type: test id: 42 body: { foo: bar } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/12_slices.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/12_slices.yml index ef5b55ad1310c..f655b43b98949 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/12_slices.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/12_slices.yml @@ -11,28 +11,24 @@ setup: - do: index: index: test_sliced_scroll - type: test id: 1 body: { foo: 1 } - do: index: index: test_sliced_scroll - type: test id: 2 body: { foo: 2 } - do: index: index: test_sliced_scroll - type: test id: 3 body: { foo: 3 } - do: index: index: test_sliced_scroll - type: test id: 4 body: { foo: 4 } @@ -149,4 +145,3 @@ setup: - do: clear_scroll: scroll_id: $scroll_id - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/20_keep_alive.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/20_keep_alive.yml index 6ebc26e3790d8..6217f66c2648e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/20_keep_alive.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/20_keep_alive.yml @@ -17,14 +17,12 @@ - do: index: index: test_scroll - type: test id: 1 body: { foo: 1 } - do: index: index: test_scroll - type: test id: 2 body: { foo: 1 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/100_avg_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/100_avg_metric.yml index 0cc2918d4c90c..559421a8ad13a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/100_avg_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/100_avg_metric.yml @@ -1,19 +1,19 @@ setup: - do: indices.create: + include_type_name: false index: test_1 body: settings: number_of_replicas: 0 mappings: - doc: - properties: - int_field: - type : integer - double_field: - type : double - string_field: - type: keyword + properties: + int_field: + type : integer + double_field: + type : double + string_field: + type: keyword - do: bulk: @@ -21,28 +21,24 @@ setup: body: - index: _index: test_1 - _type: doc _id: 1 - int_field: 1 double_field: 1.0 string_field: foo - index: _index: test_1 - _type: doc _id: 2 - int_field: 51 double_field: 51.0 string_field: foo - index: _index: test_1 - _type: doc _id: 3 - int_field: 101 double_field: 101.0 string_field: foo - index: _index: test_1 - _type: doc _id: 4 - int_field: 151 double_field: 151.0 @@ -180,4 +176,3 @@ setup: the_string_avg: avg: field: string_field - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml index 08d4ea5d6e4d9..784b95cec25cf 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml @@ -1,17 +1,17 @@ setup: - do: indices.create: + include_type_name: false index: test_1 body: settings: number_of_replicas: 0 mappings: - test: - "properties": - "number": - "type" : "integer" - "date": - "type" : "date" + "properties": + "number": + "type" : "integer" + "date": + "type" : "date" - do: cluster.health: wait_for_status: green @@ -21,131 +21,123 @@ setup: - do: index: index: test_1 - type: test id: 1 body: { "number" : 1 } - do: index: index: test_1 - type: test id: 2 body: { "number" : 51 } - do: index: index: test_1 - type: test id: 3 body: { "number" : 101 } - + - do: index: index: test_1 - type: test id: 4 body: { "number" : 151 } - + - do: indices.refresh: {} - + - do: search: rest_total_hits_as_int: true body: { "aggs" : { "histo" : { "histogram" : { "field" : "number", "interval" : 50 } } } } - + - match: { hits.total: 4 } - + - length: { aggregations.histo.buckets: 4 } - + - match: { aggregations.histo.buckets.0.key: 0 } - + - is_false: aggregations.histo.buckets.0.key_as_string - + - match: { aggregations.histo.buckets.0.doc_count: 1 } - + - match: { aggregations.histo.buckets.1.key: 50 } - + - is_false: aggregations.histo.buckets.1.key_as_string - + - match: { aggregations.histo.buckets.1.doc_count: 1 } - + - match: { aggregations.histo.buckets.2.key: 100 } - + - is_false: aggregations.histo.buckets.2.key_as_string - + - match: { aggregations.histo.buckets.2.doc_count: 1 } - + - match: { aggregations.histo.buckets.3.key: 150 } - + - is_false: aggregations.histo.buckets.3.key_as_string - + - match: { aggregations.histo.buckets.3.doc_count: 1 } - + --- "Format test": - do: index: index: test_1 - type: test id: 1 body: { "number" : 1 } - do: index: index: test_1 - type: test id: 2 body: { "number" : 51 } - do: index: index: test_1 - type: test id: 3 body: { "number" : 101 } - + - do: index: index: test_1 - type: test id: 4 body: { "number" : 151 } - + - do: indices.refresh: {} - + - do: search: rest_total_hits_as_int: true body: { "aggs" : { "histo" : { "histogram" : { "field" : "number", "interval" : 50, "format" : "Value is ##0.0" } } } } - + - match: { hits.total: 4 } - + - length: { aggregations.histo.buckets: 4 } - + - match: { aggregations.histo.buckets.0.key: 0 } - + - match: { aggregations.histo.buckets.0.key_as_string: "Value is 0.0" } - + - match: { aggregations.histo.buckets.0.doc_count: 1 } - + - match: { aggregations.histo.buckets.1.key: 50 } - + - match: { aggregations.histo.buckets.1.key_as_string: "Value is 50.0" } - + - match: { aggregations.histo.buckets.1.doc_count: 1 } - + - match: { aggregations.histo.buckets.2.key: 100 } - + - match: { aggregations.histo.buckets.2.key_as_string: "Value is 100.0" } - + - match: { aggregations.histo.buckets.2.doc_count: 1 } - + - match: { aggregations.histo.buckets.3.key: 150 } - + - match: { aggregations.histo.buckets.3.key_as_string: "Value is 150.0" } - + - match: { aggregations.histo.buckets.3.doc_count: 1 } --- @@ -158,28 +150,24 @@ setup: - do: index: index: test_1 - type: test id: 1 body: { "date" : "2016-01-01" } - do: index: index: test_1 - type: test id: 2 body: { "date" : "2016-01-02" } - do: index: index: test_1 - type: test id: 3 body: { "date" : "2016-02-01" } - do: index: index: test_1 - type: test id: 4 body: { "date" : "2016-03-01" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/110_max_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/110_max_metric.yml index ff7d4ee698825..c02dc27d3376c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/110_max_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/110_max_metric.yml @@ -1,19 +1,19 @@ setup: - do: indices.create: + include_type_name: false index: test_1 body: settings: number_of_replicas: 0 mappings: - doc: - properties: - int_field: - type : integer - double_field: - type : double - string_field: - type: keyword + properties: + int_field: + type : integer + double_field: + type : double + string_field: + type: keyword - do: bulk: @@ -21,28 +21,24 @@ setup: body: - index: _index: test_1 - _type: doc _id: 1 - int_field: 1 double_field: 1.0 string_field: foo - index: _index: test_1 - _type: doc _id: 2 - int_field: 51 double_field: 51.0 string_field: foo - index: _index: test_1 - _type: doc _id: 3 - int_field: 101 double_field: 101.0 string_field: foo - index: _index: test_1 - _type: doc _id: 4 - int_field: 151 double_field: 151.0 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/120_min_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/120_min_metric.yml index 5b6fb031312b5..a02ba6840fdee 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/120_min_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/120_min_metric.yml @@ -1,19 +1,19 @@ setup: - do: indices.create: + include_type_name: false index: test_1 body: settings: number_of_replicas: 0 mappings: - doc: - properties: - int_field: - type : integer - double_field: - type : double - string_field: - type: keyword + properties: + int_field: + type : integer + double_field: + type : double + string_field: + type: keyword - do: bulk: @@ -21,28 +21,24 @@ setup: body: - index: _index: test_1 - _type: doc _id: 1 - int_field: 1 double_field: 1.0 string_field: foo - index: _index: test_1 - _type: doc _id: 2 - int_field: 51 double_field: 51.0 string_field: foo - index: _index: test_1 - _type: doc _id: 3 - int_field: 101 double_field: 101.0 string_field: foo - index: _index: test_1 - _type: doc _id: 4 - int_field: 151 double_field: 151.0 @@ -180,4 +176,3 @@ setup: the_string_min: min: field: string_field - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/130_sum_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/130_sum_metric.yml index e80d95ca6b577..486e7f19873e7 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/130_sum_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/130_sum_metric.yml @@ -1,19 +1,19 @@ setup: - do: indices.create: + include_type_name: false index: test_1 body: settings: number_of_replicas: 0 mappings: - doc: - properties: - int_field: - type : integer - double_field: - type : double - string_field: - type: keyword + properties: + int_field: + type : integer + double_field: + type : double + string_field: + type: keyword - do: bulk: @@ -21,28 +21,24 @@ setup: body: - index: _index: test_1 - _type: doc _id: 1 - int_field: 1 double_field: 1.0 string_field: foo - index: _index: test_1 - _type: doc _id: 2 - int_field: 51 double_field: 51.0 string_field: foo - index: _index: test_1 - _type: doc _id: 3 - int_field: 101 double_field: 101.0 string_field: foo - index: _index: test_1 - _type: doc _id: 4 - int_field: 151 double_field: 151.0 @@ -180,4 +176,3 @@ setup: the_string_sum: sum: field: string_field - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/140_value_count_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/140_value_count_metric.yml index 6d4d6554b4489..d483b5e1a494f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/140_value_count_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/140_value_count_metric.yml @@ -1,19 +1,19 @@ setup: - do: indices.create: + include_type_name: false index: test_1 body: settings: number_of_replicas: 0 mappings: - doc: - properties: - int_field: - type : integer - double_field: - type : double - string_field: - type: keyword + properties: + int_field: + type : integer + double_field: + type : double + string_field: + type: keyword - do: bulk: @@ -21,28 +21,24 @@ setup: body: - index: _index: test_1 - _type: doc _id: 1 - int_field: 1 double_field: 1.0 string_field: foo - index: _index: test_1 - _type: doc _id: 2 - int_field: 51 double_field: 51.0 string_field: foo - index: _index: test_1 - _type: doc _id: 3 - int_field: 101 double_field: 101.0 string_field: foo - index: _index: test_1 - _type: doc _id: 4 - int_field: 151 double_field: 151.0 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/150_stats_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/150_stats_metric.yml index ca59de6148413..112e379c3c937 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/150_stats_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/150_stats_metric.yml @@ -1,19 +1,19 @@ setup: - do: indices.create: + include_type_name: false index: test_1 body: settings: number_of_replicas: 0 mappings: - doc: - properties: - int_field: - type : integer - double_field: - type : double - string_field: - type: keyword + properties: + int_field: + type : integer + double_field: + type : double + string_field: + type: keyword - do: bulk: @@ -21,28 +21,24 @@ setup: body: - index: _index: test_1 - _type: doc _id: 1 - int_field: 1 double_field: 1.0 string_field: foo - index: _index: test_1 - _type: doc _id: 2 - int_field: 51 double_field: 51.0 string_field: foo - index: _index: test_1 - _type: doc _id: 3 - int_field: 101 double_field: 101.0 string_field: foo - index: _index: test_1 - _type: doc _id: 4 - int_field: 151 double_field: 151.0 @@ -198,5 +194,3 @@ setup: - match: { aggregations.the_int_stats.max: 151.0 } - match: { aggregations.the_int_stats.avg: 76.0 } - match: { aggregations.the_int_stats.sum: 304.0 } - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/160_extended_stats_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/160_extended_stats_metric.yml index 599b17ff2af75..cfb6102b5f606 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/160_extended_stats_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/160_extended_stats_metric.yml @@ -1,19 +1,19 @@ setup: - do: indices.create: + include_type_name: false index: test_1 body: settings: number_of_replicas: 0 mappings: - doc: - properties: - int_field: - type : integer - double_field: - type : double - string_field: - type: keyword + properties: + int_field: + type : integer + double_field: + type : double + string_field: + type: keyword - do: bulk: @@ -21,28 +21,24 @@ setup: body: - index: _index: test_1 - _type: doc _id: 1 - int_field: 1 double_field: 1.0 string_field: foo - index: _index: test_1 - _type: doc _id: 2 - int_field: 51 double_field: 51.0 string_field: foo - index: _index: test_1 - _type: doc _id: 3 - int_field: 101 double_field: 101.0 string_field: foo - index: _index: test_1 - _type: doc _id: 4 - int_field: 151 double_field: 151.0 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/170_cardinality_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/170_cardinality_metric.yml index b493ff609db34..0ea6ef5604a31 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/170_cardinality_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/170_cardinality_metric.yml @@ -1,19 +1,19 @@ setup: - do: indices.create: + include_type_name: false index: test_1 body: settings: number_of_replicas: 0 mappings: - doc: - properties: - int_field: - type : integer - double_field: - type : double - string_field: - type: keyword + properties: + int_field: + type : integer + double_field: + type : double + string_field: + type: keyword - do: bulk: @@ -21,28 +21,24 @@ setup: body: - index: _index: test_1 - _type: doc _id: 1 - int_field: 1 double_field: 1.0 string_field: foo - index: _index: test_1 - _type: doc _id: 2 - int_field: 51 double_field: 51.0 string_field: foo - index: _index: test_1 - _type: doc _id: 3 - int_field: 101 double_field: 101.0 string_field: foo - index: _index: test_1 - _type: doc _id: 4 - int_field: 151 double_field: 151.0 @@ -217,5 +213,3 @@ setup: cardinality: field: int_field precision_threshold: -1 - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/180_percentiles_tdigest_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/180_percentiles_tdigest_metric.yml index 2f612ad83d5a1..35f07a65390a5 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/180_percentiles_tdigest_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/180_percentiles_tdigest_metric.yml @@ -1,19 +1,19 @@ setup: - do: indices.create: + include_type_name: false index: test_1 body: settings: number_of_replicas: 0 mappings: - doc: - properties: - int_field: - type : integer - double_field: - type : double - string_field: - type: keyword + properties: + int_field: + type : integer + double_field: + type : double + string_field: + type: keyword - do: bulk: @@ -21,28 +21,24 @@ setup: body: - index: _index: test_1 - _type: doc _id: 1 - int_field: 1 double_field: 1.0 string_field: foo - index: _index: test_1 - _type: doc _id: 2 - int_field: 51 double_field: 51.0 string_field: foo - index: _index: test_1 - _type: doc _id: 3 - int_field: 101 double_field: 101.0 string_field: foo - index: _index: test_1 - _type: doc _id: 4 - int_field: 151 double_field: 151.0 @@ -374,6 +370,3 @@ setup: - match: { aggregations.percentiles_int.values.1.value: 26.0 } - match: { aggregations.percentiles_int.values.2.key: 50.0 } - match: { aggregations.percentiles_int.values.2.value: 76.0 } - - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/190_percentiles_hdr_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/190_percentiles_hdr_metric.yml index 8cd758b5f356d..6e85e34538fde 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/190_percentiles_hdr_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/190_percentiles_hdr_metric.yml @@ -1,6 +1,7 @@ setup: - do: indices.create: + include_type_name: false index: test_1 body: settings: @@ -8,14 +9,13 @@ setup: number_of_shards: 5 number_of_routing_shards: 5 mappings: - doc: - properties: - int_field: - type : integer - double_field: - type : double - string_field: - type: keyword + properties: + int_field: + type : integer + double_field: + type : double + string_field: + type: keyword - do: bulk: @@ -23,28 +23,24 @@ setup: body: - index: _index: test_1 - _type: doc _id: 1 - int_field: 1 double_field: 1.0 string_field: foo - index: _index: test_1 - _type: doc _id: 2 - int_field: 51 double_field: 51.0 string_field: foo - index: _index: test_1 - _type: doc _id: 3 - int_field: 101 double_field: 101.0 string_field: foo - index: _index: test_1 - _type: doc _id: 4 - int_field: 151 double_field: 151.0 @@ -427,7 +423,6 @@ setup: - do: index: index: test_1 - type: doc id: 5 refresh: true body: { int_field: -10 } @@ -454,5 +449,3 @@ setup: - match: { aggregations.percentiles_int.values.95\.0: 151.1240234375 } - match: { aggregations.percentiles_int.values.99\.0: 151.1240234375 } - match: { _shards.failures.0.reason.type: array_index_out_of_bounds_exception } - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml index 82207fe9b3995..b77362963684c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml @@ -1,21 +1,20 @@ setup: - do: indices.create: + include_type_name: false index: my-index body: settings: number_of_shards: 1 number_of_replicas: 0 mappings: - doc: - properties: - users: - type: nested + properties: + users: + type: nested - do: index: index: my-index - type: doc id: 1 refresh: true body: | @@ -36,7 +35,6 @@ setup: - do: index: index: my-index - type: doc id: 2 refresh: true body: | diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yml index 379824211794a..d442672bf8bed 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yml @@ -1,39 +1,39 @@ setup: - do: indices.create: + include_type_name: false index: test_1 body: settings: number_of_replicas: 0 mappings: - test: - properties: - str: - type: keyword - ip: - type: ip - boolean: - type: boolean - integer: - type: long - double: - type: double - number: - type: long - date: - type: date + properties: + str: + type: keyword + ip: + type: ip + boolean: + type: boolean + integer: + type: long + double: + type: double + number: + type: long + date: + type: date - do: indices.create: + include_type_name: false index: test_2 body: settings: number_of_replicas: 0 mappings: - test: - properties: - number: - type: double + properties: + number: + type: double - do: cluster.health: @@ -44,21 +44,18 @@ setup: - do: index: index: test_1 - type: test id: 1 body: { "str" : "abc" } - do: index: index: test_1 - type: test id: 2 body: { "str": "abc" } - do: index: index: test_1 - type: test id: 3 body: { "str": "bcd" } @@ -91,21 +88,18 @@ setup: - do: index: index: test_1 - type: test id: 1 body: { "ip": "::1" } - do: index: index: test_1 - type: test id: 2 body: { "ip": "127.0.0.1" } - do: index: index: test_1 - type: test id: 3 body: { "ip": "::1" } @@ -169,21 +163,18 @@ setup: - do: index: index: test_1 - type: test id: 1 body: { "boolean": true } - do: index: index: test_1 - type: test id: 2 body: { "boolean": false } - do: index: index: test_1 - type: test id: 3 body: { "boolean": true } @@ -216,21 +207,18 @@ setup: - do: index: index: test_1 - type: test id: 1 body: { "integer": 1234 } - do: index: index: test_1 - type: test id: 2 body: { "integer": 5678 } - do: index: index: test_1 - type: test id: 3 body: { "integer": 1234 } @@ -263,21 +251,18 @@ setup: - do: index: index: test_1 - type: test id: 1 body: { "double": 1234.5 } - do: index: index: test_1 - type: test id: 2 body: { "double": 5678.5 } - do: index: index: test_1 - type: test id: 3 body: { "double": 1234.5 } @@ -310,21 +295,18 @@ setup: - do: index: index: test_1 - type: test id: 1 body: { "date": "2016-05-03" } - do: index: index: test_1 - type: test id: 2 body: { "date": "2014-09-01" } - do: index: index: test_1 - type: test id: 3 body: { "date": "2016-05-03" } @@ -384,21 +366,18 @@ setup: - do: index: index: test_1 - type: test id: 1 body: { "str" : "abc" } - do: index: index: test_1 - type: test id: 2 body: { "str": "abc" } - do: index: index: test_1 - type: test id: 3 body: { "str": "bcd" } @@ -441,21 +420,18 @@ setup: - do: index: index: test_1 - type: test id: 1 body: { "integer": 1234 } - do: index: index: test_1 - type: test id: 2 body: { "integer": 5678 } - do: index: index: test_1 - type: test id: 3 body: { "integer": 1234 } @@ -494,7 +470,6 @@ setup: - do: index: index: test_1 - type: test id: 1 body: {} @@ -520,7 +495,6 @@ setup: - do: index: index: test_1 - type: test id: 1 body: {} @@ -548,7 +522,6 @@ setup: - do: index: index: test_1 - type: test id: 1 body: {} @@ -576,7 +549,6 @@ setup: - do: index: index: test_1 - type: test id: 1 body: {} @@ -602,7 +574,6 @@ setup: - do: index: index: test_1 - type: test id: 1 body: {} @@ -628,35 +599,30 @@ setup: - do: index: index: test_1 - type: test id: 1 body: {"number": 100} - do: index: index: test_1 - type: test id: 2 body: {"number": 10} - do: index: index: test_2 - type: test id: 3 body: {"number": 100.0} - do: index: index: test_2 - type: test id: 1 body: {"number": 10.0} - do: index: index: test_2 - type: test id: 2 body: {"number": 14.6} @@ -694,21 +660,18 @@ setup: - do: index: index: test_1 - type: test id: 1 body: { "str": "abc" } - do: index: index: test_1 - type: test id: 2 body: { "str": "abc" } - do: index: index: test_1 - type: test id: 3 body: { "str": "bcd" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/220_filters_bucket.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/220_filters_bucket.yml index b7578a9ae6bc0..74e077d4c6a51 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/220_filters_bucket.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/220_filters_bucket.yml @@ -1,19 +1,19 @@ setup: - do: indices.create: + include_type_name: false index: test_1 body: settings: number_of_replicas: 0 mappings: - doc: - properties: - int_field: - type : integer - double_field: - type : double - string_field: - type: keyword + properties: + int_field: + type : integer + double_field: + type : double + string_field: + type: keyword - do: bulk: @@ -21,28 +21,24 @@ setup: body: - index: _index: test_1 - _type: doc _id: 1 - int_field: 1 double_field: 1.0 string_field: foo - index: _index: test_1 - _type: doc _id: 2 - int_field: 51 double_field: 51.0 string_field: foo - index: _index: test_1 - _type: doc _id: 3 - int_field: 101 double_field: 101.0 string_field: foo - index: _index: test_1 - _type: doc _id: 4 - int_field: 151 double_field: 151.0 @@ -265,4 +261,3 @@ setup: the_filter: filters: filters: [] - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml index 73bf44cb5d589..5e1b336e7ccf8 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml @@ -2,62 +2,56 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - doc: - properties: - date: - type: date - keyword: - type: keyword - long: - type: long - nested: - type: nested - properties: - nested_long: - type: long + properties: + date: + type: date + keyword: + type: keyword + long: + type: long + nested: + type: nested + properties: + nested_long: + type: long - do: index: index: test - type: doc id: 1 body: { "keyword": "foo", "long": [10, 20], "nested": [{"nested_long": 10}, {"nested_long": 20}] } - do: index: index: test - type: doc id: 2 body: { "keyword": ["foo", "bar"] } - do: index: index: test - type: doc id: 3 body: { "keyword": "bar", "long": [100, 0], "nested": [{"nested_long": 10}, {"nested_long": 0}] } - do: index: index: test - type: doc id: 4 body: { "keyword": "bar", "long": [1000, 0], "nested": [{"nested_long": 1000}, {"nested_long": 20}] } - do: index: index: test - type: doc id: 5 body: { "date": "2017-10-20T03:08:45" } - do: index: index: test - type: doc id: 6 body: { "date": "2017-10-21T07:00:00" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/240_max_buckets.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/240_max_buckets.yml index 0a8c951e7af7c..5ed025a72095f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/240_max_buckets.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/240_max_buckets.yml @@ -2,69 +2,61 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - doc: - properties: - keyword: - type: keyword - date: - type: date + properties: + keyword: + type: keyword + date: + type: date - do: index: index: test - type: doc id: 1 body: { "date": "2014-03-03T00:00:00", "keyword": "dgx" } - do: index: index: test - type: doc id: 2 body: { "date": "2015-03-03T00:00:00", "keyword": "dfs" } - do: index: index: test - type: doc id: 3 body: { "date": "2016-03-03T00:00:00", "keyword": "foobar" } - do: index: index: test - type: doc id: 4 body: { "date": "2017-03-03T00:00:00", "keyword": "foo" } - do: index: index: test - type: doc id: 5 body: { "date": "2018-03-03T00:00:00", "keyword": "bar" } - do: index: index: test - type: doc id: 6 body: { "date": "2019-03-03T00:00:00", "keyword": "baz" } - do: index: index: test - type: doc id: 7 body: { "date": "2020-03-03T00:00:00", "keyword": "qux" } - do: index: index: test - type: doc id: 8 body: { "date": "2021-03-03T00:00:00", "keyword": "quux" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/260_weighted_avg.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/260_weighted_avg.yml index 5d9e46a67b1d4..65f5e29f7e8e7 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/260_weighted_avg.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/260_weighted_avg.yml @@ -4,19 +4,19 @@ setup: reason: weighted_avg is only available as of 6.4.0 - do: indices.create: + include_type_name: false index: test_1 body: settings: number_of_replicas: 0 mappings: - doc: - properties: - int_field: - type : integer - double_field: - type : double - string_field: - type: keyword + properties: + int_field: + type : integer + double_field: + type : double + string_field: + type: keyword - do: bulk: @@ -24,25 +24,21 @@ setup: body: - index: _index: test_1 - _type: doc _id: 1 - int_field: 1 double_field: 1.0 - index: _index: test_1 - _type: doc _id: 2 - int_field: 2 double_field: 2.0 - index: _index: test_1 - _type: doc _id: 3 - int_field: 3 double_field: 3.0 - index: _index: test_1 - _type: doc _id: 4 - int_field: 4 double_field: 4.0 @@ -72,4 +68,3 @@ setup: - length: { hits.hits: 4 } - match: { aggregations.the_int_avg.value: 3.0 } - match: { aggregations.the_double_avg.value: 3.0 } - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/270_median_absolute_deviation_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/270_median_absolute_deviation_metric.yml index ce86adce66539..3a96000f3a5a8 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/270_median_absolute_deviation_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/270_median_absolute_deviation_metric.yml @@ -4,38 +4,35 @@ setup: reason: "added in 6.6.0" - do: indices.create: + include_type_name: false index: test body: settings: number_of_replicas: 0 mappings: - _doc: - properties: - int_field: - type: integer - double_field: - type: double - incomplete_field: - type: integer + properties: + int_field: + type: integer + double_field: + type: double + incomplete_field: + type: integer - do: bulk: refresh: true body: - index: _index: test - _type: _doc - int_field: 100 double_field: 100.0 incomplete_field: 1000 - index: _index: test - _type: _doc - int_field: 200 double_field: 200.0 incomplete_field: 2000 - index: _index: test - _type: _doc - int_field: 300 double_field: 300.0 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/30_sig_terms.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/30_sig_terms.yml index 39c83727d1bcc..3376ce4441457 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/30_sig_terms.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/30_sig_terms.yml @@ -2,59 +2,52 @@ "Default index": - do: indices.create: + include_type_name: false index: goodbad body: settings: number_of_shards: "1" mappings: - doc: - properties: - text: - type: text - fielddata: true - class: - type: keyword + properties: + text: + type: text + fielddata: true + class: + type: keyword - do: index: index: goodbad - type: doc id: 1 body: { text: "good", class: "good" } - do: index: index: goodbad - type: doc id: 2 body: { text: "good", class: "good" } - do: index: index: goodbad - type: doc id: 3 body: { text: "bad", class: "bad" } - do: index: index: goodbad - type: doc id: 4 body: { text: "bad", class: "bad" } - do: index: index: goodbad - type: doc id: 5 body: { text: "good bad", class: "good" } - do: index: index: goodbad - type: doc id: 6 body: { text: "good bad", class: "bad" } - do: index: index: goodbad - type: doc id: 7 body: { text: "bad", class: "bad" } @@ -70,7 +63,7 @@ index: goodbad - match: {hits.total: 7} - + - do: search: rest_total_hits_as_int: true @@ -79,29 +72,27 @@ - match: {aggregations.class.buckets.0.sig_terms.buckets.0.key: "bad"} - match: {aggregations.class.buckets.1.sig_terms.buckets.0.key: "good"} - + --- "IP test": - do: indices.create: + include_type_name: false index: ip_index body: mappings: - doc: - properties: - ip: - type: ip + properties: + ip: + type: ip - do: index: index: ip_index - type: doc id: 1 body: { ip: "::1" } - do: index: index: ip_index - type: doc id: 2 body: { } @@ -133,7 +124,7 @@ - length: { aggregations.ip_terms.buckets: 1 } - match: { aggregations.ip_terms.buckets.0.key: "::1" } - + - do: search: rest_total_hits_as_int: true @@ -142,11 +133,9 @@ - match: { hits.total: 1 } - length: { aggregations.ip_terms.buckets: 0 } - + - do: catch: request search: rest_total_hits_as_int: true body: { "size" : 0, "aggs" : { "ip_terms" : { "significant_terms" : { "field" : "ip", "exclude" : "127.*" } } } } - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml index c5cac876e7959..d4492861e72e8 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml @@ -1,20 +1,20 @@ setup: - do: indices.create: + include_type_name: false index: test body: settings: number_of_replicas: 0 mappings: - test: - properties: - ip: - type: ip - double: - type: double - date: - type: date - format: epoch_second + properties: + ip: + type: ip + double: + type: double + date: + type: date + format: epoch_second - do: cluster.health: @@ -25,21 +25,18 @@ setup: - do: index: index: test - type: test id: 1 body: { "double" : 42 } - do: index: index: test - type: test id: 2 body: { "double" : 100 } - do: index: index: test - type: test id: 3 body: { "double" : 50 } @@ -117,21 +114,18 @@ setup: - do: index: index: test - type: test id: 1 body: { "ip" : "::1" } - do: index: index: test - type: test id: 2 body: { "ip" : "192.168.0.1" } - do: index: index: test - type: test id: 3 body: { "ip" : "192.168.0.7" } @@ -199,13 +193,13 @@ setup: - match: { hits.total: 3 } - - length: { aggregations.ip_range.buckets: 2 } + - length: { aggregations.ip_range.buckets: 2 } - match: { aggregations.ip_range.buckets.0.key: "::/24" } - match: { aggregations.ip_range.buckets.0.to: "0:100::" } - - match: { aggregations.ip_range.buckets.0.doc_count: 3 } + - match: { aggregations.ip_range.buckets.0.doc_count: 3 } - match: { aggregations.ip_range.buckets.1.key: "192.168.0.0/16" } @@ -213,7 +207,7 @@ setup: - match: { aggregations.ip_range.buckets.1.to: "192.169.0.0" } - - match: { aggregations.ip_range.buckets.1.doc_count: 2 } + - match: { aggregations.ip_range.buckets.1.doc_count: 2 } --- "IP Range Key Generation": @@ -236,21 +230,18 @@ setup: - do: index: index: test - type: test id: 1 body: { "date" : 1000 } - do: index: index: test - type: test id: 2 body: { "date" : 2000 } - do: index: index: test - type: test id: 3 body: { "date" : 3000 } @@ -270,7 +261,7 @@ setup: - match: { aggregations.date_range.buckets.0.key: "1000-3000" } - match: { aggregations.date_range.buckets.0.from: 1000000 } - match: { aggregations.date_range.buckets.0.to: 3000000 } - + - match: { aggregations.date_range.buckets.1.doc_count: 1 } - match: { aggregations.date_range.buckets.1.key: "3000-4000" } - match: { aggregations.date_range.buckets.1.from: 3000000 } @@ -281,35 +272,30 @@ setup: - do: index: index: test - type: test id: 1 body: { "date" : "28800000000" } - do: index: index: test - type: test id: 2 body: { "date" : "315561600000" } - do: index: index: test - type: test id: 3 body: { "date" : "631180800000" } - do: index: index: test - type: test id: 4 body: { "date" : "10000" } - do: index: index: test - type: test id: 5 body: { "ip" : "192.168.0.1" } @@ -334,7 +320,7 @@ setup: to: '315561600000' - key: Other to: "200000" - + - match: { hits.total: 5 } - length: { aggregations.age_groups.buckets: 3 } @@ -350,4 +336,3 @@ setup: - match: { aggregations.age_groups.buckets.2.key: "Generation Y" } - match: { aggregations.age_groups.buckets.2.doc_count: 2 } - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/50_filter.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/50_filter.yml index 75ced7fc43dc5..6964acdab06d9 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/50_filter.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/50_filter.yml @@ -1,30 +1,28 @@ setup: - do: indices.create: + include_type_name: false index: test body: settings: number_of_shards: 1 number_of_replicas: 0 mappings: - test: - properties: - mentions: - type: keyword - notifications: - type: keyword + properties: + mentions: + type: keyword + notifications: + type: keyword - do: index: index: test - type: test id: foo|bar|baz0 body: { "notifications" : ["abc"] } - do: index: index: test - type: test id: foo|bar|baz1 body: { "mentions" : ["abc"] } @@ -34,12 +32,16 @@ setup: --- "Filter aggs with terms lookup and ensure it's cached": # Because the filter agg rewrites the terms lookup in the rewrite phase the request can be cached + - skip: + version: " - 6.99.99" + reason: types are required in requests before 7.0.0 + - do: search: rest_total_hits_as_int: true size: 0 request_cache: true - body: {"aggs": { "itemsNotify": { "filter": { "terms": { "mentions": { "index": "test", "type": "test", "id": "foo|bar|baz0", "path": "notifications"}}}, "aggs": { "mentions" : {"terms" : { "field" : "mentions" }}}}}} + body: {"aggs": { "itemsNotify": { "filter": { "terms": { "mentions": { "index": "test", "id": "foo|bar|baz0", "path": "notifications"}}}, "aggs": { "mentions" : {"terms" : { "field" : "mentions" }}}}}} # validate result - match: { hits.total: 2 } @@ -74,4 +76,3 @@ setup: - match: { _all.total.request_cache.hit_count: 0 } - match: { _all.total.request_cache.miss_count: 1 } - is_true: indices.test - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/51_filter_with_types.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/51_filter_with_types.yml new file mode 100644 index 0000000000000..3405e6390476a --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/51_filter_with_types.yml @@ -0,0 +1,57 @@ +setup: + - do: + indices.create: + include_type_name: true + index: test + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + test: + properties: + mentions: + type: keyword + notifications: + type: keyword + + - do: + index: + index: test + type: test + id: foo|bar|baz0 + body: { "notifications" : ["abc"] } + + - do: + index: + index: test + type: test + id: foo|bar|baz1 + body: { "mentions" : ["abc"] } + + - do: + indices.refresh: {} + +--- +"Filter aggs with terms lookup and ensure it's cached": + # Because the filter agg rewrites the terms lookup in the rewrite phase the request can be cached + + - do: + search: + rest_total_hits_as_int: true + size: 0 + request_cache: true + body: {"aggs": { "itemsNotify": { "filter": { "terms": { "mentions": { "index": "test", "type": "test", "id": "foo|bar|baz0", "path": "notifications"}}}, "aggs": { "mentions" : {"terms" : { "field" : "mentions" }}}}}} + + # validate result + - match: { hits.total: 2 } + - match: { aggregations.itemsNotify.doc_count: 1 } + - length: { aggregations.itemsNotify.mentions.buckets: 1 } + - match: { aggregations.itemsNotify.mentions.buckets.0.key: "abc" } + # we are using a lookup - this should not cache + - do: + indices.stats: { index: test, metric: request_cache} + - match: { _shards.total: 1 } + - match: { _all.total.request_cache.hit_count: 0 } + - match: { _all.total.request_cache.miss_count: 1 } + - is_true: indices.test diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/70_adjacency_matrix.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/70_adjacency_matrix.yml index 0ec46f10e3d65..ad364e930df89 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/70_adjacency_matrix.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/70_adjacency_matrix.yml @@ -1,35 +1,32 @@ setup: - do: indices.create: + include_type_name: false index: test body: settings: number_of_shards: 1 number_of_replicas: 0 mappings: - test: - properties: - num: - type: integer + properties: + num: + type: integer - do: index: index: test - type: test id: 1 body: { "num": [1, 2] } - do: index: index: test - type: test id: 2 body: { "num": [2, 3] } - do: index: index: test - type: test id: 3 body: { "num": [3, 4] } @@ -60,4 +57,3 @@ setup: - match: { aggregations.conns.buckets.3.doc_count: 1 } - match: { aggregations.conns.buckets.3.key: "4" } - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/80_typed_keys.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/80_typed_keys.yml index d7bb2b22ccc9e..5afd8b925e652 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/80_typed_keys.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/80_typed_keys.yml @@ -2,25 +2,24 @@ setup: - do: indices.create: + include_type_name: false index: test body: settings: number_of_replicas: 0 mappings: - test: - properties: - name: - type: keyword - num: - type: integer - created: - type: date + properties: + name: + type: keyword + num: + type: integer + created: + type: date - do: bulk: refresh: true index: test - type: test body: - '{"index": {}}' - '{"name": "one", "num": 1, "created": "2010-03-12T01:07:45"}' diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/90_sig_text.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/90_sig_text.yml index 4b1372303c175..f730a099989fe 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/90_sig_text.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/90_sig_text.yml @@ -3,59 +3,52 @@ - do: indices.create: + include_type_name: false index: goodbad body: settings: number_of_shards: "1" mappings: - doc: - properties: - text: - type: text - fielddata: false - class: - type: keyword + properties: + text: + type: text + fielddata: false + class: + type: keyword - do: index: index: goodbad - type: doc id: 1 body: { text: "good", class: "good" } - do: index: index: goodbad - type: doc id: 2 body: { text: "good", class: "good" } - do: index: index: goodbad - type: doc id: 3 body: { text: "bad", class: "bad" } - do: index: index: goodbad - type: doc id: 4 body: { text: "bad", class: "bad" } - do: index: index: goodbad - type: doc id: 5 body: { text: "good bad", class: "good" } - do: index: index: goodbad - type: doc id: 6 body: { text: "good bad", class: "bad" } - do: index: index: goodbad - type: doc id: 7 body: { text: "bad", class: "bad" } @@ -71,7 +64,7 @@ index: goodbad - match: {hits.total: 7} - + - do: search: rest_total_hits_as_int: true @@ -80,65 +73,58 @@ - match: {aggregations.class.buckets.0.sig_text.buckets.0.key: "bad"} - match: {aggregations.class.buckets.1.sig_text.buckets.0.key: "good"} - + --- "Dedup noise": - do: indices.create: + include_type_name: false index: goodbad body: settings: number_of_shards: "1" mappings: - doc: - properties: - text: - type: text - fielddata: false - class: - type: keyword + properties: + text: + type: text + fielddata: false + class: + type: keyword - do: index: index: goodbad - type: doc id: 1 body: { text: "good noisewords1 g1 g2 g3 g4 g5 g6", class: "good" } - do: index: index: goodbad - type: doc id: 2 body: { text: "good noisewords2 g1 g2 g3 g4 g5 g6", class: "good" } - do: index: index: goodbad - type: doc id: 3 body: { text: "bad noisewords3 b1 b2 b3 b4 b5 b6", class: "bad" } - do: index: index: goodbad - type: doc id: 4 body: { text: "bad noisewords4 b1 b2 b3 b4 b5 b6", class: "bad" } - do: index: index: goodbad - type: doc id: 5 body: { text: "good bad noisewords5 gb1 gb2 gb3 gb4 gb5 gb6", class: "good" } - do: index: index: goodbad - type: doc id: 6 body: { text: "good bad noisewords6 gb1 gb2 gb3 gb4 gb5 gb6", class: "bad" } - do: index: index: goodbad - type: doc id: 7 body: { text: "bad noisewords7 b1 b2 b3 b4 b5 b6", class: "bad" } @@ -154,7 +140,7 @@ index: goodbad - match: {hits.total: 7} - + - do: search: rest_total_hits_as_int: true @@ -162,7 +148,6 @@ body: {"aggs": {"class": {"terms": {"field": "class"},"aggs": {"sig_text": {"significant_text": {"field": "text", "filter_duplicate_text": true}}}}}} - match: {aggregations.class.buckets.0.sig_text.buckets.0.key: "bad"} - - length: { aggregations.class.buckets.0.sig_text.buckets: 1 } + - length: { aggregations.class.buckets.0.sig_text.buckets: 1 } - match: {aggregations.class.buckets.1.sig_text.buckets.0.key: "good"} - - length: { aggregations.class.buckets.1.sig_text.buckets: 1 } - + - length: { aggregations.class.buckets.1.sig_text.buckets: 1 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/10_unified.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/10_unified.yml index 5bb9937844ae2..03989a2da185a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/10_unified.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/10_unified.yml @@ -1,24 +1,23 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - unified: - "properties": - "text": - "type": "text" - "fields": - "fvh": - "type": "text" - "term_vector": "with_positions_offsets" - "postings": - "type": "text" - "index_options": "offsets" + "properties": + "text": + "type": "text" + "fields": + "fvh": + "type": "text" + "term_vector": "with_positions_offsets" + "postings": + "type": "text" + "index_options": "offsets" - do: index: index: test - type: unified id: 1 body: "text" : "The quick brown fox is brown." diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/20_fvh.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/20_fvh.yml index 64d9c8c28aa2e..fd1bdbc86efa4 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/20_fvh.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/20_fvh.yml @@ -1,21 +1,20 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - doc: - "properties": - "title": - "type": "text" - "term_vector": "with_positions_offsets" - "description": - "type": "text" - "term_vector": "with_positions_offsets" + "properties": + "title": + "type": "text" + "term_vector": "with_positions_offsets" + "description": + "type": "text" + "term_vector": "with_positions_offsets" - do: index: index: test - type: doc id: 1 body: "title" : "The quick brown fox is brown" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/30_max_analyzed_offset.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/30_max_analyzed_offset.yml index 840d5c849c7a1..f90d18c6d3996 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/30_max_analyzed_offset.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/30_max_analyzed_offset.yml @@ -2,24 +2,23 @@ setup: - do: indices.create: + include_type_name: false index: test1 body: settings: number_of_shards: 1 index.highlight.max_analyzed_offset: 10 mappings: - test_type: - properties: - field1: - type: text - field2: - type: text - index_options: offsets + properties: + field1: + type: text + field2: + type: text + index_options: offsets - do: index: index: test1 - type: test_type id: 1 body: "field1" : "The quick brown fox went to the forest and saw another fox." diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml index e0d61192bf754..a2b6a480d48e7 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml @@ -2,13 +2,13 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - type_1: - properties: - nested_field: - type: nested + properties: + nested_field: + type: nested --- "Nested inner hits": @@ -18,7 +18,6 @@ setup: - do: index: index: test - type: type_1 id: 1 body: "nested_field" : [ { "foo": "bar" } ] @@ -32,10 +31,10 @@ setup: body: { "query" : { "nested" : { "path" : "nested_field", "query" : { "match_all" : {} }, "inner_hits" : {} } } } - match: { hits.total: 1 } - match: { hits.hits.0._index: "test" } - - match: { hits.hits.0._type: "type_1" } + - match: { hits.hits.0._type: "_doc" } - match: { hits.hits.0._id: "1" } - match: { hits.hits.0.inner_hits.nested_field.hits.hits.0._index: "test" } - - match: { hits.hits.0.inner_hits.nested_field.hits.hits.0._type: "type_1" } + - match: { hits.hits.0.inner_hits.nested_field.hits.hits.0._type: "_doc" } - match: { hits.hits.0.inner_hits.nested_field.hits.hits.0._id: "1" } - match: { hits.hits.0.inner_hits.nested_field.hits.hits.0._nested.field: "nested_field" } - match: { hits.hits.0.inner_hits.nested_field.hits.hits.0._nested.offset: 0 } @@ -52,7 +51,6 @@ setup: - do: index: index: test - type: type_1 id: 1 body: "nested_field" : [ { "foo": "bar" } ] @@ -66,7 +64,7 @@ setup: - match: { hits.total: 1 } - match: { hits.hits.0._index: "test" } - - match: { hits.hits.0._type: "type_1" } + - match: { hits.hits.0._type: "_doc" } - match: { hits.hits.0._id: "1" } - match: { hits.hits.0._version: 1 } - match: { hits.hits.0.fields._seq_no: [0] } @@ -76,7 +74,6 @@ setup: - do: index: index: test - type: type_1 id: 1 body: "nested_field" : [ { "foo": "baz" } ] @@ -90,12 +87,9 @@ setup: - match: { hits.total: 1 } - match: { hits.hits.0._index: "test" } - - match: { hits.hits.0._type: "type_1" } + - match: { hits.hits.0._type: "_doc" } - match: { hits.hits.0._id: "1" } - match: { hits.hits.0._version: 2 } - match: { hits.hits.0.fields._seq_no: [1] } - match: { hits.hits.0.inner_hits.nested_field.hits.hits.0._version: 2 } - match: { hits.hits.0.inner_hits.nested_field.hits.hits.0.fields._seq_no: [1] } - - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/100_stored_fields.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/100_stored_fields.yml index eaff6167a057b..a82d7fff480eb 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/100_stored_fields.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/100_stored_fields.yml @@ -5,7 +5,6 @@ setup: - do: index: index: test - type: test id: 1 body: { foo: bar } - do: @@ -43,4 +42,3 @@ setup: - is_false: hits.hits.0._id - is_false: hits.hits.0._source - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yml index 18191c5ee3ac2..992d6325a1381 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yml @@ -2,19 +2,18 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - test: - properties: - bigint: - type: keyword + properties: + bigint: + type: keyword - do: index: index: test_1 - type: test id: 1 body: { "include": { "field1": "v1", "field2": "v2" }, "count": 1, "bigint": 72057594037927936 } - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml index a85abb4e6e28b..c0e998c711895 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml @@ -1,17 +1,16 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - test: properties: numeric_group: { type: integer } - do: index: index: test - type: test id: 1 version_type: external version: 11 @@ -19,7 +18,6 @@ setup: - do: index: index: test - type: test id: 2 version_type: external version: 22 @@ -27,7 +25,6 @@ setup: - do: index: index: test - type: test id: 3 version_type: external version: 33 @@ -35,7 +32,6 @@ setup: - do: index: index: test - type: test id: 4 version_type: external version: 44 @@ -43,7 +39,6 @@ setup: - do: index: index: test - type: test id: 5 version_type: external version: 55 @@ -51,7 +46,6 @@ setup: - do: index: index: test - type: test id: 6 version_type: external version: 66 @@ -74,19 +68,19 @@ setup: - match: {hits.total: 6 } - length: {hits.hits: 3 } - match: {hits.hits.0._index: test } - - match: {hits.hits.0._type: test } + - match: {hits.hits.0._type: _doc } - match: {hits.hits.0.fields.numeric_group: [3] } - match: {hits.hits.0.sort: [36] } - match: {hits.hits.0._id: "6" } - is_false: hits.hits.0.inner_hits - match: {hits.hits.1._index: test } - - match: {hits.hits.1._type: test } + - match: {hits.hits.1._type: _doc } - match: {hits.hits.1.fields.numeric_group: [1] } - match: {hits.hits.1.sort: [24] } - match: {hits.hits.1._id: "3" } - is_false: hits.hits.1.inner_hits - match: {hits.hits.2._index: test } - - match: {hits.hits.2._type: test } + - match: {hits.hits.2._type: _doc } - match: {hits.hits.2.fields.numeric_group: [25] } - match: {hits.hits.2.sort: [10] } - match: {hits.hits.2._id: "4" } @@ -107,7 +101,7 @@ setup: - match: {hits.total: 6 } - length: {hits.hits: 1 } - match: {hits.hits.0._index: test } - - match: {hits.hits.0._type: test } + - match: {hits.hits.0._type: _doc } - match: {hits.hits.0.fields.numeric_group: [25]} - match: {hits.hits.0.sort: [10] } - match: {hits.hits.0._id: "4" } @@ -127,7 +121,7 @@ setup: - match: { hits.total: 6 } - length: { hits.hits: 3 } - match: { hits.hits.0._index: test } - - match: { hits.hits.0._type: test } + - match: { hits.hits.0._type: _doc } - match: { hits.hits.0.fields.numeric_group: [3] } - match: { hits.hits.0.sort: [36] } - match: { hits.hits.0._id: "6" } @@ -135,7 +129,7 @@ setup: - length: { hits.hits.0.inner_hits.sub_hits.hits.hits: 1 } - match: { hits.hits.0.inner_hits.sub_hits.hits.hits.0._id: "6" } - match: { hits.hits.1._index: test } - - match: { hits.hits.1._type: test } + - match: { hits.hits.1._type: _doc } - match: { hits.hits.1.fields.numeric_group: [1] } - match: { hits.hits.1.sort: [24] } - match: { hits.hits.1._id: "3" } @@ -144,7 +138,7 @@ setup: - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.0._id: "2" } - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._id: "1" } - match: { hits.hits.2._index: test } - - match: { hits.hits.2._type: test } + - match: { hits.hits.2._type: _doc } - match: { hits.hits.2.fields.numeric_group: [25] } - match: { hits.hits.2.sort: [10] } - match: { hits.hits.2._id: "4" } @@ -167,7 +161,7 @@ setup: - match: { hits.total: 6 } - length: { hits.hits: 3 } - match: { hits.hits.0._index: test } - - match: { hits.hits.0._type: test } + - match: { hits.hits.0._type: _doc } - match: { hits.hits.0.fields.numeric_group: [3] } - match: { hits.hits.0.sort: [36] } - match: { hits.hits.0._id: "6" } @@ -175,7 +169,7 @@ setup: - length: { hits.hits.0.inner_hits.sub_hits.hits.hits: 1 } - match: { hits.hits.0.inner_hits.sub_hits.hits.hits.0._id: "6" } - match: { hits.hits.1._index: test } - - match: { hits.hits.1._type: test } + - match: { hits.hits.1._type: _doc } - match: { hits.hits.1.fields.numeric_group: [1] } - match: { hits.hits.1.sort: [24] } - match: { hits.hits.1._id: "3" } @@ -184,7 +178,7 @@ setup: - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.0._id: "2" } - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._id: "1" } - match: { hits.hits.2._index: test } - - match: { hits.hits.2._type: test } + - match: { hits.hits.2._type: _doc } - match: { hits.hits.2.fields.numeric_group: [25] } - match: { hits.hits.2.sort: [10] } - match: { hits.hits.2._id: "4" } @@ -278,7 +272,7 @@ setup: rest_total_hits_as_int: true index: test body: - collapse: + collapse: field: numeric_group inner_hits: - { name: sub_hits_asc, size: 2, sort: [{ sort: asc }] } @@ -288,7 +282,7 @@ setup: - match: { hits.total: 6 } - length: { hits.hits: 3 } - match: { hits.hits.0._index: test } - - match: { hits.hits.0._type: test } + - match: { hits.hits.0._type: _doc } - match: { hits.hits.0.fields.numeric_group: [3] } - match: { hits.hits.0.sort: [36] } - match: { hits.hits.0._id: "6" } @@ -299,7 +293,7 @@ setup: - length: { hits.hits.0.inner_hits.sub_hits_desc.hits.hits: 1 } - match: { hits.hits.0.inner_hits.sub_hits_desc.hits.hits.0._id: "6" } - match: { hits.hits.1._index: test } - - match: { hits.hits.1._type: test } + - match: { hits.hits.1._type: _doc } - match: { hits.hits.1.fields.numeric_group: [1] } - match: { hits.hits.1.sort: [24] } - match: { hits.hits.1._id: "3" } @@ -311,7 +305,7 @@ setup: - length: { hits.hits.1.inner_hits.sub_hits_desc.hits.hits: 1 } - match: { hits.hits.1.inner_hits.sub_hits_desc.hits.hits.0._id: "3" } - match: { hits.hits.2._index: test } - - match: { hits.hits.2._type: test } + - match: { hits.hits.2._type: _doc } - match: { hits.hits.2.fields.numeric_group: [25] } - match: { hits.hits.2.sort: [10] } - match: { hits.hits.2._id: "4" } @@ -342,7 +336,7 @@ setup: - match: { hits.total: 6 } - length: { hits.hits: 3 } - match: { hits.hits.0._index: test } - - match: { hits.hits.0._type: test } + - match: { hits.hits.0._type: _doc } - match: { hits.hits.0.fields.numeric_group: [3] } - match: { hits.hits.0.sort: [36] } - match: { hits.hits.0._id: "6" } @@ -352,7 +346,7 @@ setup: - match: { hits.hits.0.inner_hits.sub_hits.hits.hits.0._id: "6" } - match: { hits.hits.0.inner_hits.sub_hits.hits.hits.0._version: 66 } - match: { hits.hits.1._index: test } - - match: { hits.hits.1._type: test } + - match: { hits.hits.1._type: _doc } - match: { hits.hits.1.fields.numeric_group: [1] } - match: { hits.hits.1.sort: [24] } - match: { hits.hits.1._id: "3" } @@ -364,7 +358,7 @@ setup: - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._id: "1" } - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._version: 11 } - match: { hits.hits.2._index: test } - - match: { hits.hits.2._type: test } + - match: { hits.hits.2._type: _doc } - match: { hits.hits.2.fields.numeric_group: [25] } - match: { hits.hits.2.sort: [10] } - match: { hits.hits.2._id: "4" } @@ -447,7 +441,7 @@ setup: - gte: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._seq_no: 0 } - gte: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._primary_term: 1 } - match: { hits.hits.2._index: test } - - match: { hits.hits.2._type: test } + - match: { hits.hits.2._type: _doc } - match: { hits.hits.2.fields.numeric_group: [25] } - match: { hits.hits.2.sort: [10] } - match: { hits.hits.2._id: "4" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/115_multiple_field_collapsing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/115_multiple_field_collapsing.yml index 191473ebccecc..ca55785066719 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/115_multiple_field_collapsing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/115_multiple_field_collapsing.yml @@ -5,37 +5,37 @@ reason: using multiple field collapsing from 7.0 on - do: indices.create: + include_type_name: false index: addresses body: settings: number_of_shards: 1 number_of_replicas: 1 mappings: - _doc: - properties: - country: {"type": "keyword"} - city: {"type": "keyword"} - address: {"type": "text"} + properties: + country: {"type": "keyword"} + city: {"type": "keyword"} + address: {"type": "text"} - do: bulk: refresh: true body: - - '{ "index" : { "_index" : "addresses", "_type" : "_doc", "_id" : "1" } }' + - '{ "index" : { "_index" : "addresses", "_id" : "1" } }' - '{"country" : "Canada", "city" : "Saskatoon", "address" : "701 Victoria Avenue" }' - - '{ "index" : { "_index" : "addresses", "_type" : "_doc", "_id" : "2" } }' + - '{ "index" : { "_index" : "addresses", "_id" : "2" } }' - '{"country" : "Canada", "city" : "Toronto", "address" : "74 Victoria Street, Suite, 74 Victoria Street, Suite 300" }' - - '{ "index" : { "_index" : "addresses", "_type" : "_doc", "_id" : "3" } }' + - '{ "index" : { "_index" : "addresses", "_id" : "3" } }' - '{"country" : "Canada", "city" : "Toronto", "address" : "350 Victoria St" }' - - '{ "index" : { "_index" : "addresses", "_type" : "_doc", "_id" : "4" } }' + - '{ "index" : { "_index" : "addresses", "_id" : "4" } }' - '{"country" : "Canada", "city" : "Toronto", "address" : "20 Victoria Street" }' - - '{ "index" : { "_index" : "addresses", "_type" : "_doc", "_id" : "5" } }' + - '{ "index" : { "_index" : "addresses", "_id" : "5" } }' - '{"country" : "UK", "city" : "London", "address" : "58 Victoria Street" }' - - '{ "index" : { "_index" : "addresses", "_type" : "_doc", "_id" : "6" } }' + - '{ "index" : { "_index" : "addresses", "_id" : "6" } }' - '{"country" : "UK", "city" : "London", "address" : "Victoria Street Victoria Palace Theatre" }' - - '{ "index" : { "_index" : "addresses", "_type" : "_doc", "_id" : "7" } }' + - '{ "index" : { "_index" : "addresses", "_id" : "7" } }' - '{"country" : "UK", "city" : "Manchester", "address" : "75 Victoria street Westminster" }' - - '{ "index" : { "_index" : "addresses", "_type" : "_doc", "_id" : "8" } }' + - '{ "index" : { "_index" : "addresses", "_id" : "8" } }' - '{"country" : "UK", "city" : "London", "address" : "Victoria Station Victoria Arcade" }' diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/120_batch_reduce_size.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/120_batch_reduce_size.yml index df8034be98bc3..5cc22387c5118 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/120_batch_reduce_size.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/120_batch_reduce_size.yml @@ -1,16 +1,16 @@ setup: - do: indices.create: + include_type_name: false index: test_1 body: settings: number_of_shards: 5 number_of_replicas: 0 mappings: - test: - properties: - str: - type: keyword + properties: + str: + type: keyword --- "batched_reduce_size lower limit": @@ -27,21 +27,18 @@ setup: - do: index: index: test_1 - type: test id: 1 body: { "str" : "abc" } - do: index: index: test_1 - type: test id: 2 body: { "str": "abc" } - do: index: index: test_1 - type: test id: 3 body: { "str": "bcd" } - do: @@ -62,5 +59,3 @@ setup: - match: { aggregations.str_terms.buckets.1.key: "bcd" } - is_false: aggregations.str_terms.buckets.1.key_as_string - match: { aggregations.str_terms.buckets.1.doc_count: 1 } - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/140_pre_filter_search_shards.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/140_pre_filter_search_shards.yml index 0038ee4eed7eb..e0999db549127 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/140_pre_filter_search_shards.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/140_pre_filter_search_shards.yml @@ -1,40 +1,40 @@ setup: - do: indices.create: + include_type_name: false index: index_1 body: settings: number_of_shards: 1 mappings: - doc: - properties: - created_at: - type: date - format: "yyyy-MM-dd" + properties: + created_at: + type: date + format: "yyyy-MM-dd" - do: indices.create: + include_type_name: false index: index_2 body: settings: number_of_shards: 1 mappings: - doc: - properties: - created_at: - type: date - format: "yyyy-MM-dd" + properties: + created_at: + type: date + format: "yyyy-MM-dd" - do: indices.create: + include_type_name: false index: index_3 body: settings: number_of_shards: 1 mappings: - doc: - properties: - created_at: - type: date - format: "yyyy-MM-dd" + properties: + created_at: + type: date + format: "yyyy-MM-dd" --- @@ -51,20 +51,17 @@ setup: - do: index: index: index_1 - type: doc id: 1 body: { "created_at": "2016-01-01"} - do: index: index: index_2 - type: doc id: 2 body: { "created_at": "2017-01-01" } - do: index: index: index_3 - type: doc id: 3 body: { "created_at": "2018-01-01" } - do: @@ -116,8 +113,8 @@ setup: - match: { _shards.total: 3 } - match: { _shards.successful: 3 } - # skip 2 and execute one to fetch the actual empty result - - match: { _shards.skipped : 2} + # skip 2 and execute one to fetch the actual empty result + - match: { _shards.skipped : 2} - match: { _shards.failed: 0 } - match: { hits.total: 0 } @@ -159,7 +156,3 @@ setup: - match: { _shards.failed: 0 } - match: { hits.total: 2 } - length: { aggregations.idx_terms.buckets: 2 } - - - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/150_rewrite_on_coordinator.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/150_rewrite_on_coordinator.yml index 3320a8f87bb7d..626d4a8088986 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/150_rewrite_on_coordinator.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/150_rewrite_on_coordinator.yml @@ -1,33 +1,30 @@ "Ensure that we fetch the document only once": - do: indices.create: + include_type_name: false index: search_index body: settings: number_of_shards: 5 mappings: - doc: - properties: - user: - type: keyword + properties: + user: + type: keyword - do: index: index: search_index - type: doc id: 1 body: { "user": "1" } - do: index: index: search_index - type: doc id: 2 body: { "user": "2" } - do: index: index: search_index - type: doc id: 3 body: { "user": "3" } @@ -76,6 +73,3 @@ indices.stats: { index: 'lookup_index', "metric": "get"} - match: { indices.lookup_index.total.get.total: 1 } - - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/160_exists_query.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/160_exists_query.yml index 5fb06abc72162..89cc7f2135232 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/160_exists_query.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/160_exists_query.yml @@ -4,64 +4,63 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - test: - dynamic: false - properties: - binary: - type: binary - doc_values: true - boolean: - type: boolean - date: - type: date - geo_point: - type: geo_point - geo_shape: - type: geo_shape - ip: - type: ip - keyword: - type: keyword - byte: - type: byte - double: - type: double - float: - type: float - half_float: - type: half_float - integer: - type: integer - long: - type: long - short: - type: short - object: - type: object - properties: - inner1: - type: keyword - inner2: - type: keyword - text: - type: text - + dynamic: false + properties: + binary: + type: binary + doc_values: true + boolean: + type: boolean + date: + type: date + geo_point: + type: geo_point + geo_shape: + type: geo_shape + ip: + type: ip + keyword: + type: keyword + byte: + type: byte + double: + type: double + float: + type: float + half_float: + type: half_float + integer: + type: integer + long: + type: long + short: + type: short + object: + type: object + properties: + inner1: + type: keyword + inner2: + type: keyword + text: + type: text + - do: headers: Content-Type: application/json index: index: "test" - type: "test" id: 1 body: binary: "YWJjZGUxMjM0" boolean: true date: "2017-01-01" geo_point: [0.0, 20.0] - geo_shape: + geo_shape: type: "point" coordinates: [0.0, 20.0] ip: "192.168.0.1" @@ -73,24 +72,23 @@ setup: integer: 1 long: 1 short: 1 - object: + object: inner1: "foo" inner2: "bar" text: "foo bar" - + - do: headers: Content-Type: application/json index: index: "test" - type: "test" id: 2 body: binary: "YWJjZGUxMjM0" boolean: false date: "2017-01-01" geo_point: [0.0, 20.0] - geo_shape: + geo_shape: type: "point" coordinates: [0.0, 20.0] ip: "192.168.0.1" @@ -102,16 +100,15 @@ setup: integer: 1 long: 1 short: 1 - object: + object: inner1: "foo" text: "foo bar" - + - do: headers: Content-Type: application/json index: index: "test" - type: "test" id: 3 routing: "route_me" body: @@ -119,7 +116,7 @@ setup: boolean: true date: "2017-01-01" geo_point: [0.0, 20.0] - geo_shape: + geo_shape: type: "point" coordinates: [0.0, 20.0] ip: "192.168.0.1" @@ -131,93 +128,91 @@ setup: integer: 1 long: 1 short: 1 - object: + object: inner2: "bar" text: "foo bar" - + - do: index: index: "test" - type: "test" id: 4 body: {} - do: indices.create: + include_type_name: false index: test-no-dv body: mappings: - test: - dynamic: false - properties: - binary: - type: binary - doc_values: false - store: true - boolean: - type: boolean - doc_values: false - date: - type: date - doc_values: false - geo_point: - type: geo_point - doc_values: false - geo_shape: - type: geo_shape - ip: - type: ip - doc_values: false - keyword: - type: keyword - doc_values: false - byte: - type: byte - doc_values: false - double: - type: double - doc_values: false - float: - type: float - doc_values: false - half_float: - type: half_float - doc_values: false - integer: - type: integer - doc_values: false - long: - type: long - doc_values: false - short: - type: short - doc_values: false - object: - type: object - properties: - inner1: - type: keyword - doc_values: false - inner2: - type: keyword - doc_values: false - text: - type: text - doc_values: false - + dynamic: false + properties: + binary: + type: binary + doc_values: false + store: true + boolean: + type: boolean + doc_values: false + date: + type: date + doc_values: false + geo_point: + type: geo_point + doc_values: false + geo_shape: + type: geo_shape + ip: + type: ip + doc_values: false + keyword: + type: keyword + doc_values: false + byte: + type: byte + doc_values: false + double: + type: double + doc_values: false + float: + type: float + doc_values: false + half_float: + type: half_float + doc_values: false + integer: + type: integer + doc_values: false + long: + type: long + doc_values: false + short: + type: short + doc_values: false + object: + type: object + properties: + inner1: + type: keyword + doc_values: false + inner2: + type: keyword + doc_values: false + text: + type: text + doc_values: false + - do: headers: Content-Type: application/json index: index: "test-no-dv" - type: "test" id: 1 body: binary: "YWJjZGUxMjM0" boolean: true date: "2017-01-01" geo_point: [0.0, 20.0] - geo_shape: + geo_shape: type: "point" coordinates: [0.0, 20.0] ip: "192.168.0.1" @@ -229,24 +224,23 @@ setup: integer: 1 long: 1 short: 1 - object: + object: inner1: "foo" inner2: "bar" text: "foo bar" - + - do: headers: Content-Type: application/json index: index: "test-no-dv" - type: "test" id: 2 body: binary: "YWJjZGUxMjM0" boolean: false date: "2017-01-01" geo_point: [0.0, 20.0] - geo_shape: + geo_shape: type: "point" coordinates: [0.0, 20.0] ip: "192.168.0.1" @@ -258,16 +252,15 @@ setup: integer: 1 long: 1 short: 1 - object: + object: inner1: "foo" text: "foo bar" - + - do: headers: Content-Type: application/json index: index: "test-no-dv" - type: "test" id: 3 routing: "route_me" body: @@ -275,7 +268,7 @@ setup: boolean: true date: "2017-01-01" geo_point: [0.0, 20.0] - geo_shape: + geo_shape: type: "point" coordinates: [0.0, 20.0] ip: "192.168.0.1" @@ -287,79 +280,77 @@ setup: integer: 1 long: 1 short: 1 - object: + object: inner2: "bar" text: "foo bar" - + - do: index: index: "test-no-dv" - type: "test" id: 4 body: {} - do: indices.create: + include_type_name: false index: test-unmapped body: mappings: - test: - dynamic: false - properties: - unrelated: - type: keyword - + dynamic: false + properties: + unrelated: + type: keyword + - do: index: index: "test-unmapped" - type: "test" id: 1 - body: + body: unrelated: "foo" - do: indices.create: + include_type_name: false index: test-empty body: mappings: - test: - dynamic: false - properties: - binary: - type: binary - date: - type: date - geo_point: - type: geo_point - geo_shape: - type: geo_shape - ip: - type: ip - keyword: - type: keyword - byte: - type: byte - double: - type: double - float: - type: float - half_float: - type: half_float - integer: - type: integer - long: - type: long - short: - type: short - object: - type: object - properties: - inner1: - type: keyword - inner2: - type: keyword - text: - type: text + dynamic: false + properties: + binary: + type: binary + date: + type: date + geo_point: + type: geo_point + geo_shape: + type: geo_shape + ip: + type: ip + keyword: + type: keyword + byte: + type: byte + double: + type: double + float: + type: float + half_float: + type: half_float + integer: + type: integer + long: + type: long + short: + type: short + object: + type: object + properties: + inner1: + type: keyword + inner2: + type: keyword + text: + type: text - do: indices.refresh: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/170_terms_query.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/170_terms_query.yml index 3966a6a182a62..73aea9c581454 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/170_terms_query.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/170_terms_query.yml @@ -5,29 +5,29 @@ reason: index.max_terms_count setting has been added in 7.0.0 - do: indices.create: + include_type_name: false index: test_index body: settings: number_of_shards: 1 index.max_terms_count: 2 mappings: - test_type: - properties: - user: - type: keyword - followers: - type: keyword + properties: + user: + type: keyword + followers: + type: keyword - do: bulk: refresh: true body: - - '{"index": {"_index": "test_index", "_type": "test_type", "_id": "u1"}}' + - '{"index": {"_index": "test_index", "_id": "u1"}}' - '{"user": "u1", "followers": ["u2", "u3"]}' - - '{"index": {"_index": "test_index", "_type": "test_type", "_id": "u2"}}' + - '{"index": {"_index": "test_index", "_id": "u2"}}' - '{"user": "u2", "followers": ["u1", "u3", "u4"]}' - - '{"index": {"_index": "test_index", "_type": "test_type", "_id": "u3"}}' + - '{"index": {"_index": "test_index", "_id": "u3"}}' - '{"user": "u3", "followers": ["u1"]}' - - '{"index": {"_index": "test_index", "_type": "test_type", "_id": "u4"}}' + - '{"index": {"_index": "test_index", "_id": "u4"}}' - '{"user": "u4", "followers": ["u3"]}' - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml index 0e50f7f327b7b..2e0dd9339c234 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml @@ -5,24 +5,24 @@ reason: JDK9 only supports this with a special sysproperty added in 6.2.0 - do: indices.create: + include_type_name: false index: test_index body: settings: number_of_shards: 1 mappings: - doc: - properties: - date_field: - type: date - format: "E, d MMM yyyy HH:mm:ss Z" - locale: "de" + properties: + date_field: + type: date + format: "E, d MMM yyyy HH:mm:ss Z" + locale: "de" - do: bulk: refresh: true body: - - '{"index": {"_index": "test_index", "_type": "doc", "_id": "1"}}' + - '{"index": {"_index": "test_index", "_id": "1"}}' - '{"date_field": "Mi, 06 Dez 2000 02:55:00 -0800"}' - - '{"index": {"_index": "test_index", "_type": "doc", "_id": "2"}}' + - '{"index": {"_index": "test_index", "_id": "2"}}' - '{"date_field": "Do, 07 Dez 2000 02:55:00 -0800"}' - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/190_index_prefix_search.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/190_index_prefix_search.yml index 0f2d48af289c5..296db2a01dc9a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/190_index_prefix_search.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/190_index_prefix_search.yml @@ -5,21 +5,20 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - test: - properties: - text: - type: text - index_prefixes: - min_chars: 2 - max_chars: 5 + properties: + text: + type: text + index_prefixes: + min_chars: 2 + max_chars: 5 - do: index: index: test - type: test id: 1 body: { text: some short words with a stupendously long one } @@ -104,4 +103,3 @@ setup: ] - match: {hits.total: 1} - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/200_ignore_malformed.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/200_ignore_malformed.yml index cc930de47f090..70bc254f7b42c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/200_ignore_malformed.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/200_ignore_malformed.yml @@ -6,37 +6,34 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - _doc: - properties: - my_date: - type: date - ignore_malformed: true - store: true - my_ip: - type: ip - ignore_malformed: true + properties: + my_date: + type: date + ignore_malformed: true + store: true + my_ip: + type: ip + ignore_malformed: true - do: index: index: test - type: _doc id: 1 body: { "my_date": "2018-05-11", "my_ip": ":::1" } - do: index: index: test - type: _doc id: 2 body: { "my_date": "bar", "my_ip": "192.168.1.42" } - do: index: index: test - type: _doc id: 3 body: { "my_date": "bar", "my_ip": "quux" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/200_index_phrase_search.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/200_index_phrase_search.yml index 10c2869ac03af..02f4b55fd3a1c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/200_index_phrase_search.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/200_index_phrase_search.yml @@ -5,19 +5,18 @@ reason: index_phrase is only available as of 7.0.0 - do: indices.create: + include_type_name: false index: test body: mappings: - test: - properties: - text: - type: text - index_phrases: true + properties: + text: + type: text + index_phrases: true - do: index: index: test - type: test id: 1 body: { text: "peter piper picked a peck of pickled peppers" } @@ -67,5 +66,3 @@ text: "piper" - match: {hits.total: 1} - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/20_default_values.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/20_default_values.yml index db9e0f586d0e6..fd4621e48cad3 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/20_default_values.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/20_default_values.yml @@ -8,14 +8,12 @@ setup: - do: index: index: test_1 - type: test id: 1 body: { foo: bar } - do: index: index: test_2 - type: test id: 42 body: { foo: bar } @@ -48,7 +46,7 @@ setup: - match: {hits.total: 1} - match: {hits.hits.0._index: test_1 } - - match: {hits.hits.0._type: test } + - match: {hits.hits.0._type: _doc } - match: {hits.hits.0._id: "1" } - do: @@ -62,7 +60,7 @@ setup: - match: {hits.total: 1} - match: {hits.hits.0._index: test_2 } - - match: {hits.hits.0._type: test } + - match: {hits.hits.0._type: _doc } - match: {hits.hits.0._id: "42" } --- diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/210_rescore_explain.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/210_rescore_explain.yml index ad38260837b12..92bb049980dff 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/210_rescore_explain.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/210_rescore_explain.yml @@ -7,11 +7,11 @@ bulk: refresh: true body: - - '{"index": {"_index": "test_index", "_type": "_doc", "_id": "1"}}' + - '{"index": {"_index": "test_index", "_id": "1"}}' - '{"f1": "1"}' - - '{"index": {"_index": "test_index", "_type": "_doc", "_id": "2"}}' + - '{"index": {"_index": "test_index", "_id": "2"}}' - '{"f1": "2"}' - - '{"index": {"_index": "test_index", "_type": "_doc", "_id": "3"}}' + - '{"index": {"_index": "test_index", "_id": "3"}}' - '{"f1": "3"}' - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/220_total_hits_object.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/220_total_hits_object.yml index a9c37c00b929a..8823fc8922b67 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/220_total_hits_object.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/220_total_hits_object.yml @@ -10,49 +10,42 @@ setup: - do: index: index: test_1 - type: test id: 1 body: { foo: bar } - do: index: index: test_1 - type: test id: 3 body: { foo: baz } - do: index: index: test_1 - type: test id: 2 body: { foo: bar } - do: index: index: test_1 - type: test id: 4 body: { foo: bar } - do: index: index: test_2 - type: test id: 42 body: { foo: bar } - do: index: index: test_2 - type: test id: 24 body: { foo: baz } - do: index: index: test_2 - type: test id: 36 body: { foo: bar } @@ -169,5 +162,3 @@ setup: foo: bar - match: {hits.total: 2} - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/230_interval_query.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/230_interval_query.yml index 2a25055be32d0..bd7c2ded35c15 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/230_interval_query.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/230_interval_query.yml @@ -5,25 +5,25 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - test: - properties: - text: - type: text - analyzer: standard + properties: + text: + type: text + analyzer: standard - do: bulk: refresh: true body: - - '{"index": {"_index": "test", "_type": "test", "_id": "1"}}' + - '{"index": {"_index": "test", "_id": "1"}}' - '{"text" : "Some like it hot, some like it cold"}' - - '{"index": {"_index": "test", "_type": "test", "_id": "2"}}' + - '{"index": {"_index": "test", "_id": "2"}}' - '{"text" : "Its cold outside, theres no kind of atmosphere"}' - - '{"index": {"_index": "test", "_type": "test", "_id": "3"}}' + - '{"index": {"_index": "test", "_id": "3"}}' - '{"text" : "Baby its cold there outside"}' - - '{"index": {"_index": "test", "_type": "test", "_id": "4"}}' + - '{"index": {"_index": "test", "_id": "4"}}' - '{"text" : "Outside it is cold and wet"}' --- @@ -323,5 +323,3 @@ setup: query: "there" ordered: false - match: { hits.total.value: 1 } - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml index 07a871d37e65f..17735c7fd451a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml @@ -10,7 +10,6 @@ setup: - do: index: index: test_1 - type: test id: 1 body: { foo: bar } @@ -93,7 +92,7 @@ setup: body: query: match_all: {} - script_fields: + script_fields: "test1" : { "script" : { "lang": "painless", "source": "1;" }} "test2" : { "script" : { "lang": "painless", "source": "1;" }} "test3" : { "script" : { "lang": "painless", "source": "1;" }} @@ -111,7 +110,7 @@ setup: index: test_1 body: query: - regexp: + regexp: foo: "^\\[\\]\\r\\\\]|\\\\.)*\\](?:(?:\\r\\n)?[\\t])*)(?:\\.(?:(?:\\r\\n)?[\\t])*(?:[^()<>@,;:\\\\\" | .\\[\\]\\000-\\031]+(?:(?:(?:\\r\\n)?[\\t])+|\\Z|(?=[\\[\"()<>@,;:\\\\\".\\[\\]]))|\\[([^\\[\\ | ]\\r\\\\]|\\\\.)*\\](?:(?:\\r\\n)?[\\t])*))*(?:,@(?:(?:\\r\\n)?[ \\t])*(?:[^()<>@,;:\\\\\".\\ | diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/60_query_string.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/60_query_string.yml index 897f4df2985ef..7582ac0df0334 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/60_query_string.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/60_query_string.yml @@ -2,18 +2,17 @@ "search with query_string parameters": - do: indices.create: + include_type_name: false index: test body: mappings: - test: - properties: - number: - type: integer + properties: + number: + type: integer - do: index: index: test - type: test id: 1 body: { field: foo bar} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/70_response_filtering.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/70_response_filtering.yml index ec73c63fce6bb..d306cb7b1ad50 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/70_response_filtering.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/70_response_filtering.yml @@ -6,14 +6,12 @@ - do: index: index: test - type: test id: 1 body: { foo: bar } - do: index: index: test - type: test id: 2 body: { foo: bar } @@ -96,9 +94,9 @@ bulk: refresh: true body: | - {"index": {"_index": "index-1", "_type": "type-1", "_id": "1"}} + {"index": {"_index": "index-1", "_id": "1"}} {"name": "First document", "properties": {"size": 123, "meta": {"foo": "bar"}}} - {"index": {"_index": "index-1", "_type": "type-1", "_id": "2"}} + {"index": {"_index": "index-1", "_id": "2"}} {"name": "Second document", "properties": {"size": 465, "meta": {"foo": "bar", "baz": "qux"}}} - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml index d1867b9a81885..be7a45d751290 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml @@ -5,21 +5,18 @@ setup: - do: index: index: test - type: test id: 1 body: { foo: bar, age: 18 } - do: index: index: test - type: test id: 42 body: { foo: bar, age: 18 } - do: index: index: test - type: test id: 172 body: { foo: bar, age: 24 } @@ -44,7 +41,7 @@ setup: - match: {hits.total: 3 } - length: {hits.hits: 1 } - match: {hits.hits.0._index: test } - - match: {hits.hits.0._type: test } + - match: {hits.hits.0._type: _doc } - match: {hits.hits.0._id: "172" } - match: {hits.hits.0.sort: [24, "172"] } @@ -63,7 +60,7 @@ setup: - match: {hits.total: 3 } - length: {hits.hits: 1 } - match: {hits.hits.0._index: test } - - match: {hits.hits.0._type: test } + - match: {hits.hits.0._type: _doc } - match: {hits.hits.0._id: "42" } - match: {hits.hits.0.sort: [18, "42"] } @@ -82,7 +79,7 @@ setup: - match: {hits.total: 3} - length: {hits.hits: 1 } - match: {hits.hits.0._index: test } - - match: {hits.hits.0._type: test } + - match: {hits.hits.0._type: _doc } - match: {hits.hits.0._id: "1" } - match: {hits.hits.0.sort: [18, "1"] } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/issue4895.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/issue4895.yml index 2c673831388f0..4d8b1484c74ac 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/issue4895.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/issue4895.yml @@ -7,7 +7,6 @@ setup: - do: index: index: test - type: test id: 1 body: user : foo @@ -26,12 +25,9 @@ setup: search: rest_total_hits_as_int: true index: test - type: test body: query: term: data: some preference: _local stored_fields: [user,amount] - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/issue9606.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/issue9606.yml index ca75adf1c73d3..9f5025093c652 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/issue9606.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/issue9606.yml @@ -7,7 +7,6 @@ setup: - do: index: index: test - type: test id: 1 body: { foo: bar } @@ -23,7 +22,6 @@ setup: search: rest_total_hits_as_int: true index: test - type: test search_type: query_and_fetch body: query: @@ -38,7 +36,6 @@ setup: search: rest_total_hits_as_int: true index: test - type: test search_type: dfs_query_and_fetch body: query: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search_shards/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search_shards/10_basic.yml index b95b7c644e282..e1e697b496ed1 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search_shards/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search_shards/10_basic.yml @@ -15,6 +15,7 @@ "Search shards aliases with and without filters": - do: indices.create: + include_type_name: false index: test_index body: settings: @@ -22,10 +23,9 @@ number_of_shards: 1 number_of_replicas: 0 mappings: - type_1: - properties: - field: - type: text + properties: + field: + type: text aliases: test_alias_no_filter: {} test_alias_filter_1: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/10_basic.yml index 67b9f38de7784..4b6db06e29f36 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/10_basic.yml @@ -2,7 +2,6 @@ setup: - do: index: index: test - type: test id: testing_document body: body: Amsterdam meetup @@ -24,4 +23,3 @@ setup: - match: {suggest.test_suggestion.1.options.0.text: amsterdam} - match: {suggest.test_suggestion.2.options.0.text: meetup} - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/20_completion.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/20_completion.yml index ab459e2ad23ba..0afafdfe0b075 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/20_completion.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/20_completion.yml @@ -5,27 +5,27 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - test: - "properties": - "suggest_1": - "type" : "completion" - "suggest_2": - "type" : "completion" - "suggest_3": - "type" : "completion" - "suggest_4": - "type" : "completion" - "suggest_5a": - "type" : "completion" - "suggest_5b": - "type" : "completion" - "suggest_6": - "type" : "completion" - title: - type: keyword + "properties": + "suggest_1": + "type" : "completion" + "suggest_2": + "type" : "completion" + "suggest_3": + "type" : "completion" + "suggest_4": + "type" : "completion" + "suggest_5a": + "type" : "completion" + "suggest_5b": + "type" : "completion" + "suggest_6": + "type" : "completion" + title: + type: keyword --- "Simple suggestion should work": @@ -33,7 +33,6 @@ setup: - do: index: index: test - type: test id: 1 body: suggest_1: "bar" @@ -41,7 +40,6 @@ setup: - do: index: index: test - type: test id: 2 body: suggest_1: "baz" @@ -68,7 +66,6 @@ setup: - do: index: index: test - type: test id: 1 body: suggest_2: ["bar", "foo"] @@ -110,7 +107,6 @@ setup: - do: index: index: test - type: test id: 1 body: suggest_3: @@ -120,7 +116,6 @@ setup: - do: index: index: test - type: test id: 2 body: suggest_3: @@ -151,7 +146,6 @@ setup: - do: index: index: test - type: test id: 1 body: suggest_4: @@ -163,7 +157,6 @@ setup: - do: index: index: test - type: test id: 2 body: suggest_4: @@ -211,7 +204,6 @@ setup: - do: index: index: test - type: test id: 1 body: suggest_5a: "bar" @@ -254,7 +246,6 @@ setup: - do: index: index: test - type: test id: 1 body: suggest_6: @@ -266,7 +257,6 @@ setup: - do: index: index: test - type: test id: 2 body: suggest_6: @@ -292,12 +282,12 @@ setup: - length: { suggest.result.0.options: 2 } - match: { suggest.result.0.options.0.text: "baz" } - match: { suggest.result.0.options.0._index: "test" } - - match: { suggest.result.0.options.0._type: "test" } + - match: { suggest.result.0.options.0._type: "_doc" } - match: { suggest.result.0.options.0._source.title: "title_baz" } - match: { suggest.result.0.options.0._source.count: 3 } - match: { suggest.result.0.options.1.text: "bar" } - match: { suggest.result.0.options.1._index: "test" } - - match: { suggest.result.0.options.1._type: "test" } + - match: { suggest.result.0.options.1._type: "_doc" } - match: { suggest.result.0.options.1._source.title: "title_bar" } - match: { suggest.result.0.options.1._source.count: 4 } @@ -310,7 +300,6 @@ setup: - do: index: index: test - type: test id: 1 body: suggest_1: "bar" @@ -318,7 +307,6 @@ setup: - do: index: index: test - type: test id: 2 body: suggest_1: "bar" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/30_context.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/30_context.yml index 94233decd2217..1f16d5e35d6af 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/30_context.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/30_context.yml @@ -5,45 +5,45 @@ setup: - do: indices.create: + include_type_name: false index: test body: mappings: - test: - "properties": - "location": - "type": "geo_point" - "suggest_context": - "type" : "completion" - "contexts": - - - "name" : "color" - "type" : "category" - "suggest_context_with_path": - "type" : "completion" - "contexts": - - - "name" : "color" - "type" : "category" - "path" : "color" - "suggest_geo": - "type" : "completion" - "contexts": - - - "name" : "location" - "type" : "geo" - "precision" : "5km" - "suggest_multi_contexts": - "type" : "completion" - "contexts": - - - "name" : "location" - "type" : "geo" - "precision" : "5km" - "path" : "location" - - - "name" : "color" - "type" : "category" - "path" : "color" + "properties": + "location": + "type": "geo_point" + "suggest_context": + "type" : "completion" + "contexts": + - + "name" : "color" + "type" : "category" + "suggest_context_with_path": + "type" : "completion" + "contexts": + - + "name" : "color" + "type" : "category" + "path" : "color" + "suggest_geo": + "type" : "completion" + "contexts": + - + "name" : "location" + "type" : "geo" + "precision" : "5km" + "suggest_multi_contexts": + "type" : "completion" + "contexts": + - + "name" : "location" + "type" : "geo" + "precision" : "5km" + "path" : "location" + - + "name" : "color" + "type" : "category" + "path" : "color" --- "Simple context suggestion should work": @@ -51,7 +51,6 @@ setup: - do: index: index: test - type: test id: 1 body: suggest_context: @@ -62,7 +61,6 @@ setup: - do: index: index: test - type: test id: 2 body: suggest_context: @@ -95,7 +93,6 @@ setup: - do: index: index: test - type: test id: 1 body: suggest_context_with_path: @@ -106,7 +103,6 @@ setup: - do: index: index: test - type: test id: 2 body: suggest_context_with_path: "Foo blue" @@ -168,7 +164,6 @@ setup: - do: index: index: test - type: test id: 1 body: suggest_geo: @@ -181,7 +176,6 @@ setup: - do: index: index: test - type: test id: 2 body: suggest_geo: @@ -223,7 +217,6 @@ setup: - do: index: index: test - type: test id: 1 body: suggest_multi_contexts: "Marriot in Amsterdam" @@ -235,7 +228,6 @@ setup: - do: index: index: test - type: test id: 2 body: suggest_multi_contexts: "Marriot in Berlin" @@ -296,7 +288,6 @@ setup: - do: index: index: test - type: test id: 1 body: suggest_context: @@ -307,7 +298,6 @@ setup: - do: index: index: test - type: test id: 1 body: suggest_context: @@ -318,7 +308,6 @@ setup: - do: index: index: test - type: test id: 2 body: suggest_context: @@ -355,7 +344,6 @@ setup: - do: index: index: test - type: test id: 1 body: suggest_context: @@ -371,7 +359,6 @@ setup: catch: /Contexts are mandatory in context enabled completion field \[suggest_context\]/ index: index: test - type: test id: 2 body: suggest_context: @@ -379,7 +366,7 @@ setup: - do: indices.refresh: {} - + - do: catch: /Missing mandatory contexts in context query/ search: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/40_typed_keys.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/40_typed_keys.yml index 6a04997d24451..bf11481d0b900 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/40_typed_keys.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/40_typed_keys.yml @@ -2,27 +2,26 @@ setup: - do: indices.create: + include_type_name: false index: test body: settings: number_of_replicas: 0 mappings: - test: - properties: - title: - type: keyword - suggestions: - type: completion - contexts: - - - "name" : "format" - "type" : "category" + properties: + title: + type: keyword + suggestions: + type: completion + contexts: + - + "name" : "format" + "type" : "category" - do: bulk: refresh: true index: test - type: test body: - '{"index": {}}' - '{"title": "Elasticsearch in Action", "suggestions": {"input": "ELK in Action", "contexts": {"format": "ebook"}}}' diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/50_completion_with_multi_fields.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/50_completion_with_multi_fields.yml index 321ea8db2150f..fa453b9d84db2 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/50_completion_with_multi_fields.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/50_completion_with_multi_fields.yml @@ -8,21 +8,20 @@ - do: indices.create: + include_type_name: false index: completion_with_sub_keyword body: mappings: - test: - "properties": - "suggest_1": - "type" : "completion" - "fields": - "text_raw": - "type" : "keyword" + "properties": + "suggest_1": + "type" : "completion" + "fields": + "text_raw": + "type" : "keyword" - do: index: index: completion_with_sub_keyword - type: test id: 1 body: suggest_1: "bar" @@ -30,7 +29,6 @@ - do: index: index: completion_with_sub_keyword - type: test id: 2 body: suggest_1: "baz" @@ -87,7 +85,6 @@ - do: index: index: completion_with_sub_completion - type: test id: 1 body: suggest_1: "bar" @@ -95,7 +92,6 @@ - do: index: index: completion_with_sub_completion - type: test id: 2 body: suggest_1: "baz" @@ -126,30 +122,29 @@ - do: indices.create: + include_type_name: false index: completion_with_context body: mappings: - test: - "properties": - "suggest_1": - "type": "completion" - "contexts": - - - "name": "color" - "type": "category" - "fields": - "suggest_2": - "type": "completion" - "contexts": - - - "name": "color" - "type": "category" + "properties": + "suggest_1": + "type": "completion" + "contexts": + - + "name": "color" + "type": "category" + "fields": + "suggest_2": + "type": "completion" + "contexts": + - + "name": "color" + "type": "category" - do: index: index: completion_with_context - type: test id: 1 body: suggest_1: @@ -160,7 +155,6 @@ - do: index: index: completion_with_context - type: test id: 2 body: suggest_1: @@ -198,21 +192,20 @@ - do: indices.create: + include_type_name: false index: completion_with_weight body: mappings: - test: - "properties": - "suggest_1": - "type": "completion" - "fields": - "suggest_2": - "type": "completion" + "properties": + "suggest_1": + "type": "completion" + "fields": + "suggest_2": + "type": "completion" - do: index: index: completion_with_weight - type: test id: 1 body: suggest_1: @@ -222,7 +215,6 @@ - do: index: index: completion_with_weight - type: test id: 2 body: suggest_1: @@ -257,21 +249,20 @@ - do: indices.create: + include_type_name: false index: geofield_with_completion body: mappings: - test: - "properties": - "geofield": - "type": "geo_point" - "fields": - "suggest_1": - "type": "completion" + "properties": + "geofield": + "type": "geo_point" + "fields": + "suggest_1": + "type": "completion" - do: index: index: geofield_with_completion - type: test id: 1 body: geofield: "hgjhrwysvqw7" @@ -280,7 +271,6 @@ - do: index: index: geofield_with_completion - type: test id: 1 body: geofield: "hgm4psywmkn7" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/10_basic.yml index 053237f428ee2..0a4286ad3e1be 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/10_basic.yml @@ -5,18 +5,17 @@ setup: - do: indices.create: + include_type_name: false index: testidx body: mappings: - _doc: - "properties": - "text": - "type" : "text" - "term_vector" : "with_positions_offsets" + "properties": + "text": + "type" : "text" + "term_vector" : "with_positions_offsets" - do: index: index: testidx - type: _doc id: testing_document body: "text" : "The quick brown fox is brown." diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/20_issue7121.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/20_issue7121.yml index 47ba48a5f45bd..94b6413af772f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/20_issue7121.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/20_issue7121.yml @@ -7,6 +7,7 @@ setup: "Term vector API should return 'found: false' for docs between index and refresh": - do: indices.create: + include_type_name: false index: testidx body: settings: @@ -16,11 +17,10 @@ setup: number_of_replicas: 0 refresh_interval: -1 mappings: - _doc: - properties: - text: - type : "text" - term_vector : "with_positions_offsets" + properties: + text: + type : "text" + term_vector : "with_positions_offsets" - do: cluster.health: @@ -29,7 +29,6 @@ setup: - do: index: index: testidx - type: _doc id: 1 body: text : "foo bar" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/30_realtime.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/30_realtime.yml index 4e98f281bafbf..0cb6dfc06904b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/30_realtime.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/30_realtime.yml @@ -22,7 +22,6 @@ setup: - do: index: index: test_1 - type: _doc id: 1 body: { foo: bar } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/13_legacy_doc.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/13_legacy_doc.yml index b4581edd350fb..08f3457400d4f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/update/13_legacy_doc.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/update/13_legacy_doc.yml @@ -4,7 +4,6 @@ - do: index: index: test_1 - type: test id: 1 body: foo: bar @@ -14,7 +13,6 @@ - do: update: index: test_1 - type: test id: 1 body: doc: @@ -23,18 +21,16 @@ one: 3 - match: { _index: test_1 } - - match: { _type: test } + - match: { _type: _doc } - match: { _id: "1" } - match: { _version: 2 } - do: get: index: test_1 - type: test id: 1 - match: { _source.foo: baz } - match: { _source.count: 1 } - match: { _source.nested.one: 3 } - match: { _source.nested.two: 2 } - diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java index d7ff384837562..813491f2655b7 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java @@ -106,6 +106,46 @@ public ClientYamlTestResponse callApi(String apiName, Map params requestParams.put(INCLUDE_TYPE_NAME_PARAMETER, "true"); } + // When running tests against a mixed 7.x/6.x cluster we need to add the type to the document API + // requests if its not already included. + if ((apiName.equals("index") || apiName.equals("update") || apiName.equals("delete") || apiName.equals("get")) + && esVersion().before(Version.V_7_0_0) && requestParams.containsKey("type") == false) { + requestParams.put("type", "_doc"); + } + + // When running tests against a mixed 7.x/6.x cluster we need to add the type to the bulk API requests + // if its not already included. The type can either be on the request parameters or in the action metadata + // in the body of the request so we need to be sensitive to both scenarios + if (apiName.equals("bulk") && esVersion().before(Version.V_7_0_0) && requestParams.containsKey("type") == false) { + if (requestParams.containsKey("index")) { + requestParams.put("type", "_doc"); + } else { + for (int i = 0; i < bodies.size(); i++) { + Map body = bodies.get(i); + Map actionMetadata; + if (body.containsKey("index")) { + actionMetadata = (Map) body.get("index"); + i++; + } else if (body.containsKey("create")) { + actionMetadata = (Map) body.get("create"); + i++; + } else if (body.containsKey("update")) { + actionMetadata = (Map) body.get("update"); + i++; + } else if (body.containsKey("delete")) { + actionMetadata = (Map) body.get("delete"); + } else { + // action metadata is malformed so leave it malformed since + // the test is probably testing for malformed action metadata + continue; + } + if (actionMetadata.containsKey("_type") == false) { + actionMetadata.put("_type", "_doc"); + } + } + } + } + HttpEntity entity = createEntity(bodies, requestHeaders); try { response = callApiInternal(apiName, requestParams, entity, requestHeaders, nodeSelector); From e97718245d206e260a4b696565be2eb0cc0801d3 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 30 Jan 2019 11:48:34 -0500 Subject: [PATCH 056/100] Test: Enable strict deprecation on all tests (#36558) This drops the option for tests to disable strict deprecation mode in the low level rest client in favor of configuring expected warnings on any calls that should expect warnings. This behavior is paranoid-by-default which is generally the right way to handle deprecations and tests in general. --- .../rest/Netty4HeadBodyIsEmptyIT.java | 22 +++++++++---------- .../test/rest/ESRestTestCase.java | 14 +----------- 2 files changed, 12 insertions(+), 24 deletions(-) diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java index 623633f690fe0..3f7f8d7739dee 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java @@ -73,22 +73,20 @@ public void testIndexExists() throws IOException { headTestCase("/test", singletonMap("pretty", "true"), greaterThan(0)); } - @Override - protected boolean getStrictDeprecationMode() { - // Remove this override when we remove the reference to types below - return false; - } - public void testTypeExists() throws IOException { createTestDoc(); - headTestCase("/test/_mapping/_doc", emptyMap(), greaterThan(0)); - headTestCase("/test/_mapping/_doc", singletonMap("pretty", "true"), greaterThan(0)); + headTestCase("/test/_mapping/_doc", emptyMap(), OK.getStatus(), greaterThan(0), + "Type exists requests are deprecated, as types have been deprecated."); + headTestCase("/test/_mapping/_doc", singletonMap("pretty", "true"), OK.getStatus(), greaterThan(0), + "Type exists requests are deprecated, as types have been deprecated."); } public void testTypeDoesNotExist() throws IOException { createTestDoc(); - headTestCase("/test/_mapping/does-not-exist", emptyMap(), NOT_FOUND.getStatus(), greaterThan(0)); - headTestCase("/text/_mapping/_doc,does-not-exist", emptyMap(), NOT_FOUND.getStatus(), greaterThan(0)); + headTestCase("/test/_mapping/does-not-exist", emptyMap(), NOT_FOUND.getStatus(), greaterThan(0), + "Type exists requests are deprecated, as types have been deprecated."); + headTestCase("/text/_mapping/test,does-not-exist", emptyMap(), NOT_FOUND.getStatus(), greaterThan(0), + "Type exists requests are deprecated, as types have been deprecated."); } public void testAliasExists() throws IOException { @@ -193,11 +191,13 @@ private void headTestCase( final String url, final Map params, final int expectedStatusCode, - final Matcher matcher) throws IOException { + final Matcher matcher, + final String... expectedWarnings) throws IOException { Request request = new Request("HEAD", url); for (Map.Entry param : params.entrySet()) { request.addParameter(param.getKey(), param.getValue()); } + request.setOptions(expectWarnings(expectedWarnings)); Response response = client().performRequest(request); assertEquals(expectedStatusCode, response.getStatusLine().getStatusCode()); assertThat(Integer.valueOf(response.getHeader("Content-Length")), matcher); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 668f837dcedf1..23e78d5492ff7 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -692,22 +692,10 @@ protected String getProtocol() { protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException { RestClientBuilder builder = RestClient.builder(hosts); configureClient(builder, settings); - builder.setStrictDeprecationMode(getStrictDeprecationMode()); + builder.setStrictDeprecationMode(true); return builder.build(); } - /** - * Whether the used REST client should return any response containing at - * least one warning header as a failure. - * @deprecated always run in strict mode and use - * {@link RequestOptions.Builder#setWarningsHandler} to override this - * behavior on individual requests - */ - @Deprecated - protected boolean getStrictDeprecationMode() { - return true; - } - protected static void configureClient(RestClientBuilder builder, Settings settings) throws IOException { String keystorePath = settings.get(TRUSTSTORE_PATH); if (keystorePath != null) { From 2cbc6888a2dac2eb577808a379485fd8a5118bd6 Mon Sep 17 00:00:00 2001 From: Michael Basnight Date: Wed, 30 Jan 2019 11:31:59 -0600 Subject: [PATCH 057/100] HLRC: Fix strict setting exception handling (#37247) The LLRC's exception handling for strict mode was previously throwing an exception the HLRC assumed was an error response. This is not the case if the result is valid in strict mode, as it will return the proper response wrapped in an exception with warnings. This commit fixes the HLRC such that it no longer spews if it encounters a strict LLRC response. Closes #37090 --- .../client/RestHighLevelClient.java | 5 +- .../client/MockRestHighLevelTests.java | 73 +++++++++++++++++++ .../client/ResponseException.java | 4 +- .../org/elasticsearch/client/RestClient.java | 5 +- .../client/WarningFailureException.java | 58 +++++++++++++++ .../client/RestClientSingleHostTests.java | 4 +- 6 files changed, 142 insertions(+), 7 deletions(-) create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/MockRestHighLevelTests.java create mode 100644 client/rest/src/main/java/org/elasticsearch/client/WarningFailureException.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java index 5ef0e0110c12d..2eebd0cc56c66 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java @@ -1671,15 +1671,16 @@ protected final ElasticsearchStatusException parseResponseException(ResponseExce Response response = responseException.getResponse(); HttpEntity entity = response.getEntity(); ElasticsearchStatusException elasticsearchException; + RestStatus restStatus = RestStatus.fromCode(response.getStatusLine().getStatusCode()); + if (entity == null) { elasticsearchException = new ElasticsearchStatusException( - responseException.getMessage(), RestStatus.fromCode(response.getStatusLine().getStatusCode()), responseException); + responseException.getMessage(), restStatus, responseException); } else { try { elasticsearchException = parseEntity(entity, BytesRestResponse::errorFromXContent); elasticsearchException.addSuppressed(responseException); } catch (Exception e) { - RestStatus restStatus = RestStatus.fromCode(response.getStatusLine().getStatusCode()); elasticsearchException = new ElasticsearchStatusException("Unable to parse response body", restStatus, responseException); elasticsearchException.addSuppressed(e); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MockRestHighLevelTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MockRestHighLevelTests.java new file mode 100644 index 0000000000000..1c7c98cda829c --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MockRestHighLevelTests.java @@ -0,0 +1,73 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +import org.apache.http.HttpHost; +import org.apache.http.ProtocolVersion; +import org.apache.http.RequestLine; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.message.BasicRequestLine; +import org.apache.http.message.BasicStatusLine; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class MockRestHighLevelTests extends ESTestCase { + private RestHighLevelClient client; + private static final List WARNINGS = Collections.singletonList("Some Warning"); + + @Before + private void setupClient() throws IOException { + final RestClient mockClient = mock(RestClient.class); + final Response mockResponse = mock(Response.class); + + when(mockResponse.getHost()).thenReturn(new HttpHost("localhost", 9200)); + when(mockResponse.getWarnings()).thenReturn(WARNINGS); + + ProtocolVersion protocol = new ProtocolVersion("HTTP", 1, 1); + when(mockResponse.getStatusLine()).thenReturn(new BasicStatusLine(protocol, 200, "OK")); + + RequestLine requestLine = new BasicRequestLine(HttpGet.METHOD_NAME, "/_blah", protocol); + when(mockResponse.getRequestLine()).thenReturn(requestLine); + + WarningFailureException expectedException = new WarningFailureException(mockResponse); + doThrow(expectedException).when(mockClient).performRequest(any()); + + client = new RestHighLevelClient(mockClient, RestClient::close, Collections.emptyList()); + } + + public void testWarningFailure() { + WarningFailureException exception = expectThrows(WarningFailureException.class, + () -> client.info(RequestOptions.DEFAULT)); + assertThat(exception.getMessage(), equalTo("method [GET], host [http://localhost:9200], URI [/_blah], " + + "status line [HTTP/1.1 200 OK]")); + assertNull(exception.getCause()); + assertThat(exception.getResponse().getWarnings(), equalTo(WARNINGS)); + } +} diff --git a/client/rest/src/main/java/org/elasticsearch/client/ResponseException.java b/client/rest/src/main/java/org/elasticsearch/client/ResponseException.java index 0957e25fb7033..4d57f12742e03 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/ResponseException.java +++ b/client/rest/src/main/java/org/elasticsearch/client/ResponseException.java @@ -32,7 +32,7 @@ */ public final class ResponseException extends IOException { - private Response response; + private final Response response; public ResponseException(Response response) throws IOException { super(buildMessage(response)); @@ -49,7 +49,7 @@ public ResponseException(Response response) throws IOException { this.response = e.getResponse(); } - private static String buildMessage(Response response) throws IOException { + static String buildMessage(Response response) throws IOException { String message = String.format(Locale.ROOT, "method [%s], host [%s], URI [%s], status line [%s]", response.getRequestLine().getMethod(), diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java index d053bda7d44fa..175d524f02af5 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java @@ -301,7 +301,7 @@ public void completed(HttpResponse httpResponse) { if (isSuccessfulResponse(statusCode) || ignoreErrorCodes.contains(response.getStatusLine().getStatusCode())) { onResponse(node); if (thisWarningsHandler.warningsShouldFailRequest(response.getWarnings())) { - listener.onDefinitiveFailure(new ResponseException(response)); + listener.onDefinitiveFailure(new WarningFailureException(response)); } else { listener.onSuccess(response); } @@ -686,6 +686,9 @@ Response get() throws IOException { * like the asynchronous API. We wrap the exception so that the caller's * signature shows up in any exception we throw. */ + if (exception instanceof WarningFailureException) { + throw new WarningFailureException((WarningFailureException) exception); + } if (exception instanceof ResponseException) { throw new ResponseException((ResponseException) exception); } diff --git a/client/rest/src/main/java/org/elasticsearch/client/WarningFailureException.java b/client/rest/src/main/java/org/elasticsearch/client/WarningFailureException.java new file mode 100644 index 0000000000000..1cdadcc13cabc --- /dev/null +++ b/client/rest/src/main/java/org/elasticsearch/client/WarningFailureException.java @@ -0,0 +1,58 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +import java.io.IOException; + +import static org.elasticsearch.client.ResponseException.buildMessage; + +/** + * This exception is used to indicate that one or more {@link Response#getWarnings()} exist + * and is typically used when the {@link RestClient} is set to fail by setting + * {@link RestClientBuilder#setStrictDeprecationMode(boolean)} to `true`. + */ +// This class extends RuntimeException in order to deal with wrapping that is done in FutureUtils on exception. +// if the exception is not of type ElasticsearchException or RuntimeException it will be wrapped in a UncategorizedExecutionException +public final class WarningFailureException extends RuntimeException { + + private final Response response; + + public WarningFailureException(Response response) throws IOException { + super(buildMessage(response)); + this.response = response; + } + + /** + * Wrap a {@linkplain WarningFailureException} with another one with the current + * stack trace. This is used during synchronous calls so that the caller + * ends up in the stack trace of the exception thrown. + */ + WarningFailureException(WarningFailureException e) { + super(e.getMessage(), e); + this.response = e.getResponse(); + } + + /** + * Returns the {@link Response} that caused this exception to be thrown. + */ + public Response getResponse() { + return response; + } +} diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java index a37cfe87ca1c8..aaef5404f2802 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java @@ -421,9 +421,9 @@ private void assertDeprecationWarnings(List warningHeaderTexts, List Date: Wed, 30 Jan 2019 12:28:24 -0500 Subject: [PATCH 058/100] Log flush_stats and commit_stats in testMaybeFlush This test failed a few times over the last several months. It seems that we triggered a flush, but CI was too slow to finish it in several seconds. I added the flush stats and commit stats and unmuted this test. We should have a good clue if this test fails again. Relates #37896 --- .../org/elasticsearch/index/shard/IndexShardIT.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java index 36560dd96c627..787e37b37a142 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java @@ -42,6 +42,7 @@ import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.CheckedRunnable; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.bytes.BytesArray; @@ -59,6 +60,7 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.VersionType; +import org.elasticsearch.index.engine.CommitStats; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.SegmentsStats; import org.elasticsearch.index.flush.FlushStats; @@ -66,6 +68,7 @@ import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.translog.TestTranslog; import org.elasticsearch.index.translog.Translog; +import org.elasticsearch.index.translog.TranslogStats; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.breaker.CircuitBreakerStats; @@ -342,7 +345,6 @@ public void testIndexCanChangeCustomDataPath() throws Exception { assertPathHasBeenCleared(endDir.toAbsolutePath()); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37896") public void testMaybeFlush() throws Exception { createIndex("test", Settings.builder().put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.REQUEST) .build()); @@ -381,8 +383,12 @@ public void testMaybeFlush() throws Exception { logger.info("--> translog size after delete: [{}] num_ops [{}] generation [{}]", translog.stats().getUncommittedSizeInBytes(), translog.stats().getUncommittedOperations(), translog.getGeneration()); assertBusy(() -> { // this is async - logger.info("--> translog size on iter : [{}] num_ops [{}] generation [{}]", - translog.stats().getUncommittedSizeInBytes(), translog.stats().getUncommittedOperations(), translog.getGeneration()); + final TranslogStats translogStats = translog.stats(); + final CommitStats commitStats = shard.commitStats(); + final FlushStats flushStats = shard.flushStats(); + logger.info("--> translog stats [{}] gen [{}] commit_stats [{}] flush_stats [{}/{}]", + Strings.toString(translogStats), translog.getGeneration().translogFileGeneration, + commitStats.getUserData(), flushStats.getPeriodic(), flushStats.getTotal()); assertFalse(shard.shouldPeriodicallyFlush()); }); assertEquals(0, translog.stats().getUncommittedOperations()); From 9782aaa1b85b37933d08daa41388a5f47dcf3309 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Wed, 30 Jan 2019 11:56:24 -0600 Subject: [PATCH 059/100] ML: Add reason field in JobTaskState (#38029) * ML: adding reason to job failure status * marking reason as nullable * Update AutodetectProcessManager.java --- .../core/ml/job/config/JobTaskState.java | 33 +++++++++-- .../xpack/core/ml/MlTasksTests.java | 2 +- .../ml/action/TransportCloseJobAction.java | 2 +- .../autodetect/AutodetectProcessFactory.java | 3 +- .../autodetect/AutodetectProcessManager.java | 58 ++++++++++--------- .../autodetect/NativeAutodetectProcess.java | 3 +- .../NativeAutodetectProcessFactory.java | 3 +- .../normalizer/NativeNormalizerProcess.java | 2 +- .../ml/process/AbstractNativeProcess.java | 11 ++-- .../action/TransportOpenJobActionTests.java | 2 +- .../datafeed/DatafeedNodeSelectorTests.java | 2 +- .../ml/job/config/JobTaskStateTests.java | 2 +- .../AutodetectProcessManagerTests.java | 8 +-- .../NativeAutodetectProcessTests.java | 11 ++-- 14 files changed, 89 insertions(+), 53 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobTaskState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobTaskState.java index 08f73d791e53f..2651e475ff585 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobTaskState.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobTaskState.java @@ -5,6 +5,8 @@ */ package org.elasticsearch.xpack.core.ml.job.config; +import org.elasticsearch.Version; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -20,6 +22,7 @@ import java.util.Objects; import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; public class JobTaskState implements PersistentTaskState { @@ -27,9 +30,11 @@ public class JobTaskState implements PersistentTaskState { private static ParseField STATE = new ParseField("state"); private static ParseField ALLOCATION_ID = new ParseField("allocation_id"); + private static ParseField REASON = new ParseField("reason"); private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>(NAME, true, args -> new JobTaskState((JobState) args[0], (Long) args[1])); + new ConstructingObjectParser<>(NAME, true, + args -> new JobTaskState((JobState) args[0], (Long) args[1], (String) args[2])); static { PARSER.declareField(constructorArg(), p -> { @@ -39,6 +44,7 @@ public class JobTaskState implements PersistentTaskState { throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); }, STATE, ObjectParser.ValueType.STRING); PARSER.declareLong(constructorArg(), ALLOCATION_ID); + PARSER.declareString(optionalConstructorArg(), REASON); } public static JobTaskState fromXContent(XContentParser parser) { @@ -51,21 +57,33 @@ public static JobTaskState fromXContent(XContentParser parser) { private final JobState state; private final long allocationId; + private final String reason; - public JobTaskState(JobState state, long allocationId) { + public JobTaskState(JobState state, long allocationId, @Nullable String reason) { this.state = Objects.requireNonNull(state); this.allocationId = allocationId; + this.reason = reason; } public JobTaskState(StreamInput in) throws IOException { state = JobState.fromStream(in); allocationId = in.readLong(); + if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + reason = in.readOptionalString(); + } else { + reason = null; + } } public JobState getState() { return state; } + @Nullable + public String getReason() { + return reason; + } + /** * The job state stores the allocation ID at the time it was last set. * This method compares the allocation ID in the state with the allocation @@ -90,6 +108,9 @@ public String getWriteableName() { public void writeTo(StreamOutput out) throws IOException { state.writeTo(out); out.writeLong(allocationId); + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + out.writeOptionalString(reason); + } } @Override @@ -102,6 +123,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); builder.field(STATE.getPreferredName(), state.value()); builder.field(ALLOCATION_ID.getPreferredName(), allocationId); + if (reason != null) { + builder.field(REASON.getPreferredName(), reason); + } builder.endObject(); return builder; } @@ -112,11 +136,12 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; JobTaskState that = (JobTaskState) o; return state == that.state && - Objects.equals(allocationId, that.allocationId); + Objects.equals(allocationId, that.allocationId) && + Objects.equals(reason, that.reason); } @Override public int hashCode() { - return Objects.hash(state, allocationId); + return Objects.hash(state, allocationId, reason); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java index e80b47b057bf7..3afe76b8b171f 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java @@ -33,7 +33,7 @@ public void testGetJobState() { new PersistentTasksCustomMetaData.Assignment("bar", "test assignment")); assertEquals(JobState.OPENING, MlTasks.getJobState("foo", tasksBuilder.build())); - tasksBuilder.updateTaskState(MlTasks.jobTaskId("foo"), new JobTaskState(JobState.OPENED, tasksBuilder.getLastAllocationId())); + tasksBuilder.updateTaskState(MlTasks.jobTaskId("foo"), new JobTaskState(JobState.OPENED, tasksBuilder.getLastAllocationId(), null)); assertEquals(JobState.OPENED, MlTasks.getJobState("foo", tasksBuilder.build())); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java index 3149928f6af74..1076533660273 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java @@ -266,7 +266,7 @@ static TransportCloseJobAction.WaitForCloseRequest buildWaitForCloseRequest(List @Override protected void taskOperation(CloseJobAction.Request request, TransportOpenJobAction.JobTask jobTask, ActionListener listener) { - JobTaskState taskState = new JobTaskState(JobState.CLOSING, jobTask.getAllocationId()); + JobTaskState taskState = new JobTaskState(JobState.CLOSING, jobTask.getAllocationId(), "close job (api)"); jobTask.updatePersistentTaskState(taskState, ActionListener.wrap(task -> { // we need to fork because we are now on a network threadpool and closeJob method may take a while to complete: threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME).execute(new AbstractRunnable() { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessFactory.java index c95e3a5f6e340..d76593eea8997 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessFactory.java @@ -9,6 +9,7 @@ import org.elasticsearch.xpack.ml.job.process.autodetect.params.AutodetectParams; import java.util.concurrent.ExecutorService; +import java.util.function.Consumer; /** * Factory interface for creating implementations of {@link AutodetectProcess} @@ -28,5 +29,5 @@ public interface AutodetectProcessFactory { AutodetectProcess createAutodetectProcess(Job job, AutodetectParams autodetectParams, ExecutorService executorService, - Runnable onProcessCrash); + Consumer onProcessCrash); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java index 1d8f4f273601f..6b8eada7406f6 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java @@ -475,14 +475,14 @@ protected void doRun() { .kill(); processByAllocation.remove(jobTask.getAllocationId()); } finally { - setJobState(jobTask, JobState.FAILED, e2 -> closeHandler.accept(e1, true)); + setJobState(jobTask, JobState.FAILED, e1.getMessage(), e2 -> closeHandler.accept(e1, true)); } } } }); }, e1 -> { logger.warn("Failed to gather information required to open job [" + jobId + "]", e1); - setJobState(jobTask, JobState.FAILED, e2 -> closeHandler.accept(e1, true)); + setJobState(jobTask, JobState.FAILED, e1.getMessage(), e2 -> closeHandler.accept(e1, true)); }); }, e -> closeHandler.accept(e, true) @@ -601,8 +601,8 @@ private void notifyLoadingSnapshot(String jobId, AutodetectParams autodetectPara auditor.info(jobId, msg); } - private Runnable onProcessCrash(JobTask jobTask) { - return () -> { + private Consumer onProcessCrash(JobTask jobTask) { + return (reason) -> { ProcessContext processContext = processByAllocation.remove(jobTask.getAllocationId()); if (processContext != null) { AutodetectCommunicator communicator = processContext.getAutodetectCommunicator(); @@ -610,7 +610,7 @@ private Runnable onProcessCrash(JobTask jobTask) { communicator.destroyCategorizationAnalyzer(); } } - setJobState(jobTask, JobState.FAILED); + setJobState(jobTask, JobState.FAILED, reason); try { removeTmpStorage(jobTask.getJobId()); } catch (IOException e) { @@ -666,7 +666,7 @@ public void closeJob(JobTask jobTask, boolean restart, String reason) { throw e; } logger.warn("[" + jobId + "] Exception closing autodetect process", e); - setJobState(jobTask, JobState.FAILED); + setJobState(jobTask, JobState.FAILED, e.getMessage()); throw ExceptionsHelper.serverError("Exception closing autodetect process", e); } finally { // to ensure the contract that multiple simultaneous close calls for the same job wait until @@ -720,8 +720,8 @@ public Optional jobOpenTime(JobTask jobTask) { return Optional.of(Duration.between(communicator.getProcessStartTime(), ZonedDateTime.now())); } - void setJobState(JobTask jobTask, JobState state) { - JobTaskState jobTaskState = new JobTaskState(state, jobTask.getAllocationId()); + void setJobState(JobTask jobTask, JobState state, String reason) { + JobTaskState jobTaskState = new JobTaskState(state, jobTask.getAllocationId(), reason); jobTask.updatePersistentTaskState(jobTaskState, new ActionListener>() { @Override public void onResponse(PersistentTask persistentTask) { @@ -735,27 +735,31 @@ public void onFailure(Exception e) { }); } - void setJobState(JobTask jobTask, JobState state, CheckedConsumer handler) { - JobTaskState jobTaskState = new JobTaskState(state, jobTask.getAllocationId()); + void setJobState(JobTask jobTask, JobState state) { + setJobState(jobTask, state, null); + } + + void setJobState(JobTask jobTask, JobState state, String reason, CheckedConsumer handler) { + JobTaskState jobTaskState = new JobTaskState(state, jobTask.getAllocationId(), reason); jobTask.updatePersistentTaskState(jobTaskState, new ActionListener>() { - @Override - public void onResponse(PersistentTask persistentTask) { - try { - handler.accept(null); - } catch (IOException e1) { - logger.warn("Error while delegating response", e1); - } - } + @Override + public void onResponse(PersistentTask persistentTask) { + try { + handler.accept(null); + } catch (IOException e1) { + logger.warn("Error while delegating response", e1); + } + } - @Override - public void onFailure(Exception e) { - try { - handler.accept(e); - } catch (IOException e1) { - logger.warn("Error while delegating exception [" + e.getMessage() + "]", e1); - } - } - }); + @Override + public void onFailure(Exception e) { + try { + handler.accept(e); + } catch (IOException e1) { + logger.warn("Error while delegating exception [" + e.getMessage() + "]", e1); + } + } + }); } public Optional> getStatistics(JobTask jobTask) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcess.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcess.java index 69ed0d66c8606..96d5a74097532 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcess.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcess.java @@ -28,6 +28,7 @@ import java.nio.file.Path; import java.util.Iterator; import java.util.List; +import java.util.function.Consumer; /** * Autodetect process using native code. @@ -42,7 +43,7 @@ class NativeAutodetectProcess extends AbstractNativeProcess implements Autodetec NativeAutodetectProcess(String jobId, InputStream logStream, OutputStream processInStream, InputStream processOutStream, OutputStream processRestoreStream, int numberOfFields, List filesToDelete, - AutodetectResultsParser resultsParser, Runnable onProcessCrash) { + AutodetectResultsParser resultsParser, Consumer onProcessCrash) { super(jobId, logStream, processInStream, processOutStream, processRestoreStream, numberOfFields, filesToDelete, onProcessCrash); this.resultsParser = resultsParser; } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessFactory.java index 3185ebc6f1c7d..27bf1dd675325 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessFactory.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.Objects; import java.util.concurrent.ExecutorService; +import java.util.function.Consumer; public class NativeAutodetectProcessFactory implements AutodetectProcessFactory { @@ -56,7 +57,7 @@ public NativeAutodetectProcessFactory(Environment env, Settings settings, Native public AutodetectProcess createAutodetectProcess(Job job, AutodetectParams params, ExecutorService executorService, - Runnable onProcessCrash) { + Consumer onProcessCrash) { List filesToDelete = new ArrayList<>(); ProcessPipes processPipes = new ProcessPipes(env, NAMED_PIPE_HELPER, AutodetectBuilder.AUTODETECT, job.getId(), true, false, true, true, params.modelSnapshot() != null, diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcess.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcess.java index 6b67ffa6acb6f..ec22d35f16872 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcess.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcess.java @@ -20,7 +20,7 @@ class NativeNormalizerProcess extends AbstractNativeProcess implements Normalize private static final String NAME = "normalizer"; NativeNormalizerProcess(String jobId, InputStream logStream, OutputStream processInStream, InputStream processOutStream) { - super(jobId, logStream, processInStream, processOutStream, null, 0, Collections.emptyList(), () -> {}); + super(jobId, logStream, processInStream, processOutStream, null, 0, Collections.emptyList(), (ignore) -> {}); } @Override diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/AbstractNativeProcess.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/AbstractNativeProcess.java index b84bfdd38e19a..25e671a6de1e9 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/AbstractNativeProcess.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/AbstractNativeProcess.java @@ -23,12 +23,14 @@ import java.time.Duration; import java.time.ZonedDateTime; import java.util.List; +import java.util.Locale; import java.util.Objects; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.function.Consumer; /** * Abstract class for implementing a native process. @@ -48,7 +50,7 @@ public abstract class AbstractNativeProcess implements NativeProcess { private final ZonedDateTime startTime; private final int numberOfFields; private final List filesToDelete; - private final Runnable onProcessCrash; + private final Consumer onProcessCrash; private volatile Future logTailFuture; private volatile Future stateProcessorFuture; private volatile boolean processCloseInitiated; @@ -57,7 +59,7 @@ public abstract class AbstractNativeProcess implements NativeProcess { protected AbstractNativeProcess(String jobId, InputStream logStream, OutputStream processInStream, InputStream processOutStream, OutputStream processRestoreStream, int numberOfFields, List filesToDelete, - Runnable onProcessCrash) { + Consumer onProcessCrash) { this.jobId = jobId; cppLogHandler = new CppLogMessageHandler(jobId, logStream); this.processInStream = new BufferedOutputStream(processInStream); @@ -90,8 +92,9 @@ public void start(ExecutorService executorService) { // by a user or other process (e.g. the Linux OOM killer) String errors = cppLogHandler.getErrors(); - LOGGER.error("[{}] {} process stopped unexpectedly: {}", jobId, getName(), errors); - onProcessCrash.run(); + String fullError = String.format(Locale.ROOT, "[%s] %s process stopped unexpectedly: %s", jobId, getName(), errors); + LOGGER.error(fullError); + onProcessCrash.accept(fullError); } } }); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java index b23e042609030..9b7673338f619 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java @@ -552,7 +552,7 @@ public static void addJobTask(String jobId, String nodeId, JobState jobState, Pe new Assignment(nodeId, "test assignment")); if (jobState != null) { builder.updateTaskState(MlTasks.jobTaskId(jobId), - new JobTaskState(jobState, builder.getLastAllocationId() - (isStale ? 1 : 0))); + new JobTaskState(jobState, builder.getLastAllocationId() - (isStale ? 1 : 0), null)); } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java index 4b81cbb2dd6d3..39f3a3f4889c1 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java @@ -240,7 +240,7 @@ public void testSelectNode_jobTaskStale() { PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); addJobTask(job.getId(), nodeId, JobState.OPENED, tasksBuilder); // Set to lower allocationId, so job task is stale: - tasksBuilder.updateTaskState(MlTasks.jobTaskId(job.getId()), new JobTaskState(JobState.OPENED, 0)); + tasksBuilder.updateTaskState(MlTasks.jobTaskId(job.getId()), new JobTaskState(JobState.OPENED, 0, null)); tasks = tasksBuilder.build(); givenClusterState("foo", 1, 0); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobTaskStateTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobTaskStateTests.java index 26560f1034f9e..7048c4b5d2dfe 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobTaskStateTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobTaskStateTests.java @@ -15,7 +15,7 @@ public class JobTaskStateTests extends AbstractSerializingTestCase @Override protected JobTaskState createTestInstance() { - return new JobTaskState(randomFrom(JobState.values()), randomLong()); + return new JobTaskState(randomFrom(JobState.values()), randomLong(), randomAlphaOfLength(10)); } @Override diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java index 6b4fd270b1bb7..14b6d08514fad 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java @@ -204,7 +204,7 @@ public void testOpenJob() { manager.openJob(jobTask, clusterState, (e, b) -> {}); assertEquals(1, manager.numberOfOpenJobs()); assertTrue(manager.jobHasActiveAutodetectProcess(jobTask)); - verify(jobTask).updatePersistentTaskState(eq(new JobTaskState(JobState.OPENED, 1L)), any()); + verify(jobTask).updatePersistentTaskState(eq(new JobTaskState(JobState.OPENED, 1L, null)), any()); } @@ -266,10 +266,10 @@ public void testOpenJob_exceedMaxNumJobs() { doReturn(executorService).when(manager).createAutodetectExecutorService(any()); doAnswer(invocationOnMock -> { - CheckedConsumer consumer = (CheckedConsumer) invocationOnMock.getArguments()[2]; + CheckedConsumer consumer = (CheckedConsumer) invocationOnMock.getArguments()[3]; consumer.accept(null); return null; - }).when(manager).setJobState(any(), eq(JobState.FAILED), any()); + }).when(manager).setJobState(any(), eq(JobState.FAILED), any(), any()); JobTask jobTask = mock(JobTask.class); when(jobTask.getJobId()).thenReturn("foo"); @@ -512,7 +512,7 @@ public void testCloseThrows() { expectThrows(ElasticsearchException.class, () -> manager.closeJob(jobTask, false, null)); assertEquals(0, manager.numberOfOpenJobs()); - verify(manager).setJobState(any(), eq(JobState.FAILED)); + verify(manager).setJobState(any(), eq(JobState.FAILED), any()); } public void testWriteUpdateProcessMessage() { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessTests.java index 8542061c761a2..3f1275142b968 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessTests.java @@ -29,6 +29,7 @@ import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import java.util.function.Consumer; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -58,7 +59,7 @@ public void testProcessStartTime() throws Exception { try (NativeAutodetectProcess process = new NativeAutodetectProcess("foo", logStream, mock(OutputStream.class), outputStream, mock(OutputStream.class), NUMBER_FIELDS, null, - new AutodetectResultsParser(), mock(Runnable.class))) { + new AutodetectResultsParser(), mock(Consumer.class))) { process.start(executorService, mock(AutodetectStateProcessor.class), mock(InputStream.class)); ZonedDateTime startTime = process.getProcessStartTime(); @@ -80,7 +81,7 @@ public void testWriteRecord() throws IOException { ByteArrayOutputStream bos = new ByteArrayOutputStream(1024); try (NativeAutodetectProcess process = new NativeAutodetectProcess("foo", logStream, bos, outputStream, mock(OutputStream.class), NUMBER_FIELDS, Collections.emptyList(), - new AutodetectResultsParser(), mock(Runnable.class))) { + new AutodetectResultsParser(), mock(Consumer.class))) { process.start(executorService, mock(AutodetectStateProcessor.class), mock(InputStream.class)); process.writeRecord(record); @@ -114,7 +115,7 @@ public void testFlush() throws IOException { ByteArrayOutputStream bos = new ByteArrayOutputStream(AutodetectControlMsgWriter.FLUSH_SPACES_LENGTH + 1024); try (NativeAutodetectProcess process = new NativeAutodetectProcess("foo", logStream, bos, outputStream, mock(OutputStream.class), NUMBER_FIELDS, Collections.emptyList(), - new AutodetectResultsParser(), mock(Runnable.class))) { + new AutodetectResultsParser(), mock(Consumer.class))) { process.start(executorService, mock(AutodetectStateProcessor.class), mock(InputStream.class)); FlushJobParams params = FlushJobParams.builder().build(); @@ -147,7 +148,7 @@ public void testConsumeAndCloseOutputStream() throws IOException { try (NativeAutodetectProcess process = new NativeAutodetectProcess("foo", logStream, processInStream, processOutStream, mock(OutputStream.class), NUMBER_FIELDS, Collections.emptyList(), - new AutodetectResultsParser(), mock(Runnable.class))) { + new AutodetectResultsParser(), mock(Consumer.class))) { process.consumeAndCloseOutputStream(); assertThat(processOutStream.available(), equalTo(0)); @@ -162,7 +163,7 @@ private void testWriteMessage(CheckedConsumer writeFunc ByteArrayOutputStream bos = new ByteArrayOutputStream(1024); try (NativeAutodetectProcess process = new NativeAutodetectProcess("foo", logStream, bos, outputStream, mock(OutputStream.class), NUMBER_FIELDS, Collections.emptyList(), - new AutodetectResultsParser(), mock(Runnable.class))) { + new AutodetectResultsParser(), mock(Consumer.class))) { process.start(executorService, mock(AutodetectStateProcessor.class), mock(InputStream.class)); writeFunction.accept(process); From c468b2f7caff023d285e8a01e39a3873973eaf28 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 30 Jan 2019 12:56:58 -0500 Subject: [PATCH 060/100] Make primary terms fields private in index shard (#38036) This commit encapsulates the primary terms fields in index shard. This is a precursor to pushing the operation primary term down to the replication tracker. --- .../elasticsearch/index/shard/IndexShard.java | 4 +-- .../index/shard/IndexShardTests.java | 35 +++++++++++-------- .../index/shard/ShardGetServiceTests.java | 2 +- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index dc43d42c94a5c..772607903e61d 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -198,8 +198,8 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl protected volatile ShardRouting shardRouting; protected volatile IndexShardState state; - protected volatile long pendingPrimaryTerm; // see JavaDocs for getPendingPrimaryTerm - protected volatile long operationPrimaryTerm; + private volatile long pendingPrimaryTerm; // see JavaDocs for getPendingPrimaryTerm + private volatile long operationPrimaryTerm; protected final AtomicReference currentEngineReference = new AtomicReference<>(); final EngineFactory engineFactory; diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 1ea90ec6e8fbc..26bc4c964eec2 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -956,18 +956,18 @@ private void finish() { // our operation should be blocked until the previous operations complete assertFalse(onResponse.get()); assertNull(onFailure.get()); - assertThat(indexShard.operationPrimaryTerm, equalTo(primaryTerm)); + assertThat(indexShard.getOperationPrimaryTerm(), equalTo(primaryTerm)); assertThat(TestTranslog.getCurrentTerm(getTranslog(indexShard)), equalTo(primaryTerm)); Releasables.close(operation1); // our operation should still be blocked assertFalse(onResponse.get()); assertNull(onFailure.get()); - assertThat(indexShard.operationPrimaryTerm, equalTo(primaryTerm)); + assertThat(indexShard.getOperationPrimaryTerm(), equalTo(primaryTerm)); assertThat(TestTranslog.getCurrentTerm(getTranslog(indexShard)), equalTo(primaryTerm)); Releasables.close(operation2); barrier.await(); // now lock acquisition should have succeeded - assertThat(indexShard.operationPrimaryTerm, equalTo(newPrimaryTerm)); + assertThat(indexShard.getOperationPrimaryTerm(), equalTo(newPrimaryTerm)); assertThat(indexShard.getPendingPrimaryTerm(), equalTo(newPrimaryTerm)); assertThat(TestTranslog.getCurrentTerm(getTranslog(indexShard)), equalTo(newPrimaryTerm)); if (engineClosed) { @@ -1008,7 +1008,7 @@ public void onFailure(Exception e) { } }; - final long oldPrimaryTerm = indexShard.pendingPrimaryTerm - 1; + final long oldPrimaryTerm = indexShard.getPendingPrimaryTerm() - 1; randomReplicaOperationPermitAcquisition(indexShard, oldPrimaryTerm, indexShard.getGlobalCheckpoint(), randomNonNegativeLong(), onLockAcquired, ""); latch.await(); @@ -1030,7 +1030,7 @@ public void testAcquireReplicaPermitAdvanceMaxSeqNoOfUpdates() throws Exception long newMaxSeqNoOfUpdates = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE); PlainActionFuture fut = new PlainActionFuture<>(); - randomReplicaOperationPermitAcquisition(replica, replica.operationPrimaryTerm, replica.getGlobalCheckpoint(), + randomReplicaOperationPermitAcquisition(replica, replica.getOperationPrimaryTerm(), replica.getGlobalCheckpoint(), newMaxSeqNoOfUpdates, fut, ""); try (Releasable ignored = fut.actionGet()) { assertThat(replica.getMaxSeqNoOfUpdatesOrDeletes(), equalTo(Math.max(currentMaxSeqNoOfUpdates, newMaxSeqNoOfUpdates))); @@ -1181,7 +1181,7 @@ public void testRollbackReplicaEngineOnPromotion() throws IOException, Interrupt final Engine beforeRollbackEngine = indexShard.getEngine(); final long newMaxSeqNoOfUpdates = randomLongBetween(indexShard.getMaxSeqNoOfUpdatesOrDeletes(), Long.MAX_VALUE); randomReplicaOperationPermitAcquisition(indexShard, - indexShard.pendingPrimaryTerm + 1, + indexShard.getPendingPrimaryTerm() + 1, globalCheckpoint, newMaxSeqNoOfUpdates, new ActionListener() { @@ -2105,10 +2105,6 @@ public void testRecoverFromStoreRemoveStaleOperations() throws Exception { new SourceToParse(indexName, "_doc", "doc-1", new BytesArray("{}"), XContentType.JSON)); flushShard(shard); assertThat(getShardDocUIDs(shard), containsInAnyOrder("doc-0", "doc-1")); - // Here we try to increase term (i.e. a new primary is promoted) without rolling back a replica so we can keep stale operations - // in the index commit; then verify that a recovery from store (started with the safe commit) will remove all stale operations. - shard.pendingPrimaryTerm++; - shard.operationPrimaryTerm++; shard.getEngine().rollTranslogGeneration(); shard.markSeqNoAsNoop(1, "test"); shard.applyIndexOperationOnReplica(2, 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false, @@ -2118,11 +2114,20 @@ public void testRecoverFromStoreRemoveStaleOperations() throws Exception { closeShard(shard, false); // Recovering from store should discard doc #1 final ShardRouting replicaRouting = shard.routingEntry(); - IndexShard newShard = reinitShard(shard, - newShardRouting(replicaRouting.shardId(), replicaRouting.currentNodeId(), true, ShardRoutingState.INITIALIZING, - RecoverySource.ExistingStoreRecoverySource.INSTANCE)); - newShard.pendingPrimaryTerm++; - newShard.operationPrimaryTerm++; + final IndexMetaData newShardIndexMetadata = IndexMetaData.builder(shard.indexSettings().getIndexMetaData()) + .primaryTerm(replicaRouting.shardId().id(), shard.getOperationPrimaryTerm() + 1) + .build(); + closeShards(shard); + IndexShard newShard = newShard( + newShardRouting(replicaRouting.shardId(), replicaRouting.currentNodeId(), true, ShardRoutingState.INITIALIZING, + RecoverySource.ExistingStoreRecoverySource.INSTANCE), + shard.shardPath(), + newShardIndexMetadata, + null, + null, + shard.getEngineFactory(), + shard.getGlobalCheckpointSyncer(), + EMPTY_EVENT_LISTENER); DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); newShard.markAsRecovering("store", new RecoveryState(newShard.routingEntry(), localNode, null)); assertTrue(newShard.recoverFromStore()); diff --git a/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java b/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java index 14e513ff89cfe..496221ca9fc4e 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java @@ -83,7 +83,7 @@ public void testGetForUpdate() throws IOException { assertTrue(testGet1.getFields().containsKey(RoutingFieldMapper.NAME)); assertEquals("foobar", testGet1.getFields().get(RoutingFieldMapper.NAME).getValue()); - final long primaryTerm = primary.operationPrimaryTerm; + final long primaryTerm = primary.getOperationPrimaryTerm(); testGet1 = primary.getService().getForUpdate("test", "1", MATCH_ANY, VersionType.INTERNAL, test2.getSeqNo(), primaryTerm); assertEquals(new String(testGet1.source(), StandardCharsets.UTF_8), "{\"foo\" : \"baz\"}"); From 6500b0cbd78a0e3968ecb54a33b59a2325059ff9 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 30 Jan 2019 13:20:40 -0500 Subject: [PATCH 061/100] Expose retention leases in shard stats (#37991) This commit exposes retention leases via shard-level stats. --- .../stats/TransportClusterStatsAction.java | 19 ++- .../admin/indices/stats/ShardStats.java | 35 ++++- .../stats/TransportIndicesStatsAction.java | 15 ++- .../index/seqno/RetentionLease.java | 12 ++ .../index/seqno/RetentionLeaseStats.java | 124 ++++++++++++++++++ .../elasticsearch/index/shard/IndexShard.java | 6 + .../elasticsearch/indices/IndicesService.java | 23 ++-- .../elasticsearch/cluster/DiskUsageTests.java | 4 +- .../index/seqno/RetentionLeaseStatsTests.java | 67 ++++++++++ ...tentionLeaseStatsWireSerializingTests.java | 55 ++++++++ .../index/seqno/RetentionLeaseSyncIT.java | 13 +- .../shard/IndexShardRetentionLeaseTests.java | 55 +++++++- .../index/shard/IndexShardTests.java | 9 +- .../action/cat/RestIndicesActionTests.java | 2 +- .../IndicesStatsMonitoringDocTests.java | 6 +- 15 files changed, 401 insertions(+), 44 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseStats.java create mode 100644 server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseStatsTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseStatsWireSerializingTests.java diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java index a5c4adc53c42a..4cf81c24fbf1a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -37,6 +37,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.CommitStats; +import org.elasticsearch.index.seqno.RetentionLeaseStats; import org.elasticsearch.index.seqno.SeqNoStats; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.indices.IndicesService; @@ -101,21 +102,25 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq // only report on fully started shards CommitStats commitStats; SeqNoStats seqNoStats; + RetentionLeaseStats retentionLeaseStats; try { commitStats = indexShard.commitStats(); seqNoStats = indexShard.seqNoStats(); - } catch (AlreadyClosedException e) { + retentionLeaseStats = indexShard.getRetentionLeaseStats(); + } catch (final AlreadyClosedException e) { // shard is closed - no stats is fine commitStats = null; seqNoStats = null; + retentionLeaseStats = null; } shardsStats.add( - new ShardStats( - indexShard.routingEntry(), - indexShard.shardPath(), - new CommonStats(indicesService.getIndicesQueryCache(), indexShard, SHARD_STATS_FLAGS), - commitStats, - seqNoStats)); + new ShardStats( + indexShard.routingEntry(), + indexShard.shardPath(), + new CommonStats(indicesService.getIndicesQueryCache(), indexShard, SHARD_STATS_FLAGS), + commitStats, + seqNoStats, + retentionLeaseStats)); } } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java index 898f3d69456b0..9297447695a61 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java @@ -26,22 +26,36 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.ToXContent.Params; import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.engine.CommitStats; +import org.elasticsearch.index.seqno.RetentionLeaseStats; import org.elasticsearch.index.seqno.SeqNoStats; import org.elasticsearch.index.shard.ShardPath; import java.io.IOException; public class ShardStats implements Streamable, Writeable, ToXContentFragment { + private ShardRouting shardRouting; private CommonStats commonStats; @Nullable private CommitStats commitStats; @Nullable private SeqNoStats seqNoStats; + + @Nullable + private RetentionLeaseStats retentionLeaseStats; + + /** + * Gets the current retention lease stats. + * + * @return the current retention lease stats + */ + public RetentionLeaseStats getRetentionLeaseStats() { + return retentionLeaseStats; + } + private String dataPath; private String statePath; private boolean isCustomDataPath; @@ -49,7 +63,13 @@ public class ShardStats implements Streamable, Writeable, ToXContentFragment { ShardStats() { } - public ShardStats(ShardRouting routing, ShardPath shardPath, CommonStats commonStats, CommitStats commitStats, SeqNoStats seqNoStats) { + public ShardStats( + final ShardRouting routing, + final ShardPath shardPath, + final CommonStats commonStats, + final CommitStats commitStats, + final SeqNoStats seqNoStats, + final RetentionLeaseStats retentionLeaseStats) { this.shardRouting = routing; this.dataPath = shardPath.getRootDataPath().toString(); this.statePath = shardPath.getRootStatePath().toString(); @@ -57,6 +77,7 @@ public ShardStats(ShardRouting routing, ShardPath shardPath, CommonStats commonS this.commitStats = commitStats; this.commonStats = commonStats; this.seqNoStats = seqNoStats; + this.retentionLeaseStats = retentionLeaseStats; } /** @@ -109,6 +130,9 @@ public void readFrom(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { seqNoStats = in.readOptionalWriteable(SeqNoStats::new); } + if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + retentionLeaseStats = in.readOptionalWriteable(RetentionLeaseStats::new); + } } @Override @@ -122,6 +146,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { out.writeOptionalWriteable(seqNoStats); } + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + out.writeOptionalWriteable(retentionLeaseStats); + } } @Override @@ -140,6 +167,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (seqNoStats != null) { seqNoStats.toXContent(builder, params); } + if (retentionLeaseStats != null) { + retentionLeaseStats.toXContent(builder, params); + } builder.startObject(Fields.SHARD_PATH); builder.field(Fields.STATE_PATH, statePath); builder.field(Fields.DATA_PATH, dataPath); @@ -159,4 +189,5 @@ static final class Fields { static final String NODE = "node"; static final String RELOCATING_NODE = "relocating_node"; } + } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java index d339184c5f814..8371023738b3b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.CommitStats; +import org.elasticsearch.index.seqno.RetentionLeaseStats; import org.elasticsearch.index.seqno.SeqNoStats; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardNotFoundException; @@ -106,15 +107,23 @@ protected ShardStats shardOperation(IndicesStatsRequest request, ShardRouting sh CommonStats commonStats = new CommonStats(indicesService.getIndicesQueryCache(), indexShard, request.flags()); CommitStats commitStats; SeqNoStats seqNoStats; + RetentionLeaseStats retentionLeaseStats; try { commitStats = indexShard.commitStats(); seqNoStats = indexShard.seqNoStats(); - } catch (AlreadyClosedException e) { + retentionLeaseStats = indexShard.getRetentionLeaseStats(); + } catch (final AlreadyClosedException e) { // shard is closed - no stats is fine commitStats = null; seqNoStats = null; + retentionLeaseStats = null; } - return new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), commonStats, - commitStats, seqNoStats); + return new ShardStats( + indexShard.routingEntry(), + indexShard.shardPath(), + commonStats, + commitStats, + seqNoStats, + retentionLeaseStats); } } diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLease.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLease.java index 24d144d810d9c..362d068f45e2d 100644 --- a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLease.java +++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLease.java @@ -28,7 +28,9 @@ import java.util.Collection; import java.util.Collections; import java.util.Locale; +import java.util.Map; import java.util.Objects; +import java.util.function.Function; import java.util.stream.Collectors; /** @@ -242,4 +244,14 @@ public String toString() { '}'; } + /** + * A utility method to convert a collection of retention leases to a map from retention lease ID to retention lease. + * + * @param leases the leases + * @return the map from retention lease ID to retention lease + */ + static Map toMap(final Collection leases) { + return leases.stream().collect(Collectors.toMap(RetentionLease::id, Function.identity())); + } + } diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseStats.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseStats.java new file mode 100644 index 0000000000000..b8f1454a12c2b --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseStats.java @@ -0,0 +1,124 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.seqno; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Collection; +import java.util.Objects; + +/** + * Represents retention lease stats. + */ +public final class RetentionLeaseStats implements ToXContentFragment, Writeable { + + private final Collection leases; + + /** + * The underlying retention leases backing this stats object. + * + * @return the leases + */ + public Collection leases() { + return leases; + } + + /** + * Constructs a new retention lease stats object from the specified leases. + * + * @param leases the leases + */ + public RetentionLeaseStats(final Collection leases) { + this.leases = Objects.requireNonNull(leases); + } + + /** + * Constructs a new retention lease stats object from a stream. The retention lease stats should have been written via + * {@link #writeTo(StreamOutput)}. + * + * @param in the stream to construct the retention lease stats from + * @throws IOException if an I/O exception occurs reading from the stream + */ + public RetentionLeaseStats(final StreamInput in) throws IOException { + leases = in.readList(RetentionLease::new); + } + + /** + * Writes a retention lease stats object to a stream in a manner suitable for later reconstruction via + * {@link #RetentionLeaseStats(StreamInput)} (StreamInput)}. + * + * @param out the stream to write the retention lease stats to + * @throws IOException if an I/O exception occurs writing to the stream + */ + @Override + public void writeTo(final StreamOutput out) throws IOException { + out.writeCollection(leases); + } + + /** + * Converts the retention lease stats to {@link org.elasticsearch.common.xcontent.XContent} using the specified builder and pararms. + * + * @param builder the builder + * @param params the params + * @return the builder that these retention leases were converted to {@link org.elasticsearch.common.xcontent.XContent} into + * @throws IOException if an I/O exception occurs writing to the builder + */ + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + builder.startObject("retention_leases"); + { + builder.startArray("leases"); + { + for (final RetentionLease retentionLease : leases) { + builder.startObject(); + { + builder.field("id", retentionLease.id()); + builder.field("retaining_seq_no", retentionLease.retainingSequenceNumber()); + builder.field("timestamp", retentionLease.timestamp()); + builder.field("source", retentionLease.source()); + } + builder.endObject(); + } + } + builder.endArray(); + } + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + final RetentionLeaseStats that = (RetentionLeaseStats) o; + return Objects.equals(leases, that.leases); + } + + @Override + public int hashCode() { + return Objects.hash(leases); + } + +} diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 772607903e61d..446b21269d5a5 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -108,6 +108,7 @@ import org.elasticsearch.index.search.stats.ShardSearchStats; import org.elasticsearch.index.seqno.ReplicationTracker; import org.elasticsearch.index.seqno.RetentionLease; +import org.elasticsearch.index.seqno.RetentionLeaseStats; import org.elasticsearch.index.seqno.SeqNoStats; import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.PrimaryReplicaSyncer.ResyncTask; @@ -1895,6 +1896,11 @@ public Collection getRetentionLeases() { return replicationTracker.getRetentionLeases(); } + public RetentionLeaseStats getRetentionLeaseStats() { + verifyNotClosed(); + return new RetentionLeaseStats(getRetentionLeases()); + } + /** * Adds a new retention lease. * diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index cca63c015f1c7..9113ba3f23b14 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -95,6 +95,7 @@ import org.elasticsearch.index.recovery.RecoveryStats; import org.elasticsearch.index.refresh.RefreshStats; import org.elasticsearch.index.search.stats.SearchStats; +import org.elasticsearch.index.seqno.RetentionLeaseStats; import org.elasticsearch.index.seqno.RetentionLeaseSyncer; import org.elasticsearch.index.seqno.SeqNoStats; import org.elasticsearch.index.shard.IllegalIndexShardStateException; @@ -367,23 +368,29 @@ IndexShardStats indexShardStats(final IndicesService indicesService, final Index CommitStats commitStats; SeqNoStats seqNoStats; + RetentionLeaseStats retentionLeaseStats; try { commitStats = indexShard.commitStats(); seqNoStats = indexShard.seqNoStats(); + retentionLeaseStats = indexShard.getRetentionLeaseStats(); } catch (AlreadyClosedException e) { // shard is closed - no stats is fine commitStats = null; seqNoStats = null; + retentionLeaseStats = null; } - return new IndexShardStats(indexShard.shardId(), - new ShardStats[] { - new ShardStats(indexShard.routingEntry(), - indexShard.shardPath(), - new CommonStats(indicesService.getIndicesQueryCache(), indexShard, flags), - commitStats, - seqNoStats) - }); + return new IndexShardStats( + indexShard.shardId(), + new ShardStats[]{ + new ShardStats( + indexShard.routingEntry(), + indexShard.shardPath(), + new CommonStats(indicesService.getIndicesQueryCache(), indexShard, flags), + commitStats, + seqNoStats, + retentionLeaseStats) + }); } /** diff --git a/server/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java b/server/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java index c4fcb9bdb53e2..fcccaf6c0f021 100644 --- a/server/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java @@ -113,8 +113,8 @@ public void testFillShardLevelInfo() { CommonStats commonStats1 = new CommonStats(); commonStats1.store = new StoreStats(1000); ShardStats[] stats = new ShardStats[] { - new ShardStats(test_0, new ShardPath(false, test0Path, test0Path, test_0.shardId()), commonStats0 , null, null), - new ShardStats(test_1, new ShardPath(false, test1Path, test1Path, test_1.shardId()), commonStats1 , null, null) + new ShardStats(test_0, new ShardPath(false, test0Path, test0Path, test_0.shardId()), commonStats0 , null, null, null), + new ShardStats(test_1, new ShardPath(false, test1Path, test1Path, test_1.shardId()), commonStats1 , null, null, null) }; ImmutableOpenMap.Builder shardSizes = ImmutableOpenMap.builder(); ImmutableOpenMap.Builder routingToPath = ImmutableOpenMap.builder(); diff --git a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseStatsTests.java b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseStatsTests.java new file mode 100644 index 0000000000000..d77acc53b247e --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseStatsTests.java @@ -0,0 +1,67 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.seqno; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.action.support.replication.ReplicationResponse; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.test.ESSingleNodeTestCase; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; + +import static org.hamcrest.Matchers.arrayWithSize; +import static org.hamcrest.Matchers.equalTo; + +public class RetentionLeaseStatsTests extends ESSingleNodeTestCase { + + public void testRetentionLeaseStats() throws InterruptedException { + final Settings settings = Settings.builder() + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .build(); + createIndex("index", settings); + ensureGreen("index"); + final IndexShard primary = + node().injector().getInstance(IndicesService.class).getShardOrNull(new ShardId(resolveIndex("index"), 0)); + final int length = randomIntBetween(0, 8); + final Map currentRetentionLeases = new HashMap<>(); + for (int i = 0; i < length; i++) { + final String id = randomValueOtherThanMany(currentRetentionLeases.keySet()::contains, () -> randomAlphaOfLength(8)); + final long retainingSequenceNumber = randomLongBetween(0, Long.MAX_VALUE); + final String source = randomAlphaOfLength(8); + final CountDownLatch latch = new CountDownLatch(1); + final ActionListener listener = ActionListener.wrap(r -> latch.countDown(), e -> fail(e.toString())); + currentRetentionLeases.put(id, primary.addRetentionLease(id, retainingSequenceNumber, source, listener)); + latch.await(); + } + + final IndicesStatsResponse indicesStats = client().admin().indices().prepareStats("index").execute().actionGet(); + assertThat(indicesStats.getShards(), arrayWithSize(1)); + final RetentionLeaseStats retentionLeaseStats = indicesStats.getShards()[0].getRetentionLeaseStats(); + assertThat(RetentionLease.toMap(retentionLeaseStats.leases()), equalTo(currentRetentionLeases)); + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseStatsWireSerializingTests.java b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseStatsWireSerializingTests.java new file mode 100644 index 0000000000000..fe5dee782c4fe --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseStatsWireSerializingTests.java @@ -0,0 +1,55 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.seqno; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; + +public class RetentionLeaseStatsWireSerializingTests extends AbstractWireSerializingTestCase { + + @Override + protected RetentionLeaseStats createTestInstance() { + final int length = randomIntBetween(0, 8); + final Collection leases; + if (length == 0) { + leases = Collections.emptyList(); + } else { + leases = new ArrayList<>(length); + for (int i = 0; i < length; i++) { + final String id = randomAlphaOfLength(8); + final long retainingSequenceNumber = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE); + final long timestamp = randomNonNegativeLong(); + final String source = randomAlphaOfLength(8); + leases.add(new RetentionLease(id, retainingSequenceNumber, timestamp, source)); + } + } + return new RetentionLeaseStats(leases); + } + + @Override + protected Writeable.Reader instanceReader() { + return RetentionLeaseStats::new; + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncIT.java b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncIT.java index a99d0caea8e6c..f2819bfe161eb 100644 --- a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncIT.java +++ b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncIT.java @@ -37,8 +37,6 @@ import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import java.util.stream.Collectors; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.contains; @@ -46,6 +44,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItem; +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST) public class RetentionLeaseSyncIT extends ESIntegTestCase { public void testRetentionLeasesSyncedOnAdd() throws Exception { @@ -77,7 +76,7 @@ public void testRetentionLeasesSyncedOnAdd() throws Exception { // check retention leases have been committed on the primary final Collection primaryCommittedRetentionLeases = RetentionLease.decodeRetentionLeases( primary.acquireLastIndexCommit(false).getIndexCommit().getUserData().get(Engine.RETENTION_LEASES)); - assertThat(currentRetentionLeases, equalTo(toMap(primaryCommittedRetentionLeases))); + assertThat(currentRetentionLeases, equalTo(RetentionLease.toMap(primaryCommittedRetentionLeases))); // check current retention leases have been synced to all replicas for (final ShardRouting replicaShard : clusterService().state().routingTable().index("index").shard(0).replicaShards()) { @@ -86,13 +85,13 @@ public void testRetentionLeasesSyncedOnAdd() throws Exception { final IndexShard replica = internalCluster() .getInstance(IndicesService.class, replicaShardNodeName) .getShardOrNull(new ShardId(resolveIndex("index"), 0)); - final Map retentionLeasesOnReplica = toMap(replica.getRetentionLeases()); + final Map retentionLeasesOnReplica = RetentionLease.toMap(replica.getRetentionLeases()); assertThat(retentionLeasesOnReplica, equalTo(currentRetentionLeases)); // check retention leases have been committed on the replica final Collection replicaCommittedRetentionLeases = RetentionLease.decodeRetentionLeases( replica.acquireLastIndexCommit(false).getIndexCommit().getUserData().get(Engine.RETENTION_LEASES)); - assertThat(currentRetentionLeases, equalTo(toMap(replicaCommittedRetentionLeases))); + assertThat(currentRetentionLeases, equalTo(RetentionLease.toMap(replicaCommittedRetentionLeases))); } } } @@ -164,8 +163,4 @@ public void testRetentionLeasesSyncOnExpiration() throws Exception { } } - private static Map toMap(final Collection replicaCommittedRetentionLeases) { - return replicaCommittedRetentionLeases.stream().collect(Collectors.toMap(RetentionLease::id, Function.identity())); - } - } diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardRetentionLeaseTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardRetentionLeaseTests.java index 26e67fb6dd264..f66b383c2799c 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardRetentionLeaseTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardRetentionLeaseTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.InternalEngineFactory; import org.elasticsearch.index.seqno.RetentionLease; +import org.elasticsearch.index.seqno.RetentionLeaseStats; import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.threadpool.ThreadPool; @@ -81,7 +82,8 @@ public void testAddOrRenewRetentionLease() throws IOException { for (int i = 0; i < length; i++) { minimumRetainingSequenceNumbers[i] = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE); indexShard.addRetentionLease( - Integer.toString(i), minimumRetainingSequenceNumbers[i], "test-" + i, ActionListener.wrap(() -> {})); + Integer.toString(i), minimumRetainingSequenceNumbers[i], "test-" + i, ActionListener.wrap(() -> { + })); assertRetentionLeases(indexShard, i + 1, minimumRetainingSequenceNumbers, () -> 0L, true); } @@ -115,7 +117,8 @@ private void runExpirationTest(final boolean primary) throws IOException { final long[] retainingSequenceNumbers = new long[1]; retainingSequenceNumbers[0] = randomLongBetween(0, Long.MAX_VALUE); if (primary) { - indexShard.addRetentionLease("0", retainingSequenceNumbers[0], "test-0", ActionListener.wrap(() -> {})); + indexShard.addRetentionLease("0", retainingSequenceNumbers[0], "test-0", ActionListener.wrap(() -> { + })); } else { indexShard.updateRetentionLeasesOnReplica( Collections.singleton(new RetentionLease("0", retainingSequenceNumbers[0], currentTimeMillis.get(), "test-0"))); @@ -176,7 +179,8 @@ public void testCommit() throws IOException { minimumRetainingSequenceNumbers[i] = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE); currentTimeMillis.set(TimeUnit.NANOSECONDS.toMillis(randomNonNegativeLong())); indexShard.addRetentionLease( - Integer.toString(i), minimumRetainingSequenceNumbers[i], "test-" + i, ActionListener.wrap(() -> {})); + Integer.toString(i), minimumRetainingSequenceNumbers[i], "test-" + i, ActionListener.wrap(() -> { + })); } currentTimeMillis.set(TimeUnit.NANOSECONDS.toMillis(Long.MAX_VALUE)); @@ -215,13 +219,52 @@ public void testCommit() throws IOException { } } + public void testRetentionLeaseStats() throws IOException { + final IndexShard indexShard = newStartedShard(true); + try { + final int length = randomIntBetween(0, 8); + final long[] minimumRetainingSequenceNumbers = new long[length]; + for (int i = 0; i < length; i++) { + minimumRetainingSequenceNumbers[i] = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE); + indexShard.addRetentionLease( + Integer.toString(i), minimumRetainingSequenceNumbers[i], "test-" + i, ActionListener.wrap(() -> { + })); + } + final RetentionLeaseStats stats = indexShard.getRetentionLeaseStats(); + assertRetentionLeases( + stats.leases(), + indexShard.indexSettings().getRetentionLeaseMillis(), + length, + minimumRetainingSequenceNumbers, + () -> 0L, + true); + } finally { + closeShards(indexShard); + } + } + private void assertRetentionLeases( final IndexShard indexShard, final int size, final long[] minimumRetainingSequenceNumbers, final LongSupplier currentTimeMillisSupplier, final boolean primary) { - final Collection retentionLeases = indexShard.getEngine().config().retentionLeasesSupplier().get(); + assertRetentionLeases( + indexShard.getEngine().config().retentionLeasesSupplier().get(), + indexShard.indexSettings().getRetentionLeaseMillis(), + size, + minimumRetainingSequenceNumbers, + currentTimeMillisSupplier, + primary); + } + + private void assertRetentionLeases( + final Collection retentionLeases, + final long retentionLeaseMillis, + final int size, + final long[] minimumRetainingSequenceNumbers, + final LongSupplier currentTimeMillisSupplier, + final boolean primary) { final Map idToRetentionLease = new HashMap<>(); for (final RetentionLease retentionLease : retentionLeases) { idToRetentionLease.put(retentionLease.id(), retentionLease); @@ -234,9 +277,7 @@ private void assertRetentionLeases( assertThat(retentionLease.retainingSequenceNumber(), equalTo(minimumRetainingSequenceNumbers[i])); if (primary) { // retention leases can be expired on replicas, so we can only assert on primaries here - assertThat( - currentTimeMillisSupplier.getAsLong() - retentionLease.timestamp(), - lessThanOrEqualTo(indexShard.indexSettings().getRetentionLeaseMillis())); + assertThat(currentTimeMillisSupplier.getAsLong() - retentionLease.timestamp(), lessThanOrEqualTo(retentionLeaseMillis)); } assertThat(retentionLease.source(), equalTo("test-" + i)); } diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 26bc4c964eec2..12a7fad466e29 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -1385,8 +1385,13 @@ public void testMinimumCompatVersion() throws IOException { public void testShardStats() throws IOException { IndexShard shard = newStartedShard(); - ShardStats stats = new ShardStats(shard.routingEntry(), shard.shardPath(), - new CommonStats(new IndicesQueryCache(Settings.EMPTY), shard, new CommonStatsFlags()), shard.commitStats(), shard.seqNoStats()); + ShardStats stats = new ShardStats( + shard.routingEntry(), + shard.shardPath(), + new CommonStats(new IndicesQueryCache(Settings.EMPTY), shard, new CommonStatsFlags()), + shard.commitStats(), + shard.seqNoStats(), + shard.getRetentionLeaseStats()); assertEquals(shard.shardPath().getRootDataPath().toString(), stats.getDataPath()); assertEquals(shard.shardPath().getRootStatePath().toString(), stats.getStatePath()); assertEquals(shard.shardPath().isCustomDataPath(), stats.isCustomDataPath()); diff --git a/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java index 13e94f7fe5368..83bb8b309a7cb 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java @@ -163,7 +163,7 @@ private IndicesStatsResponse randomIndicesStatsResponse(final Index[] indices) { stats.get = new GetStats(); stats.flush = new FlushStats(); stats.warmer = new WarmerStats(); - shardStats.add(new ShardStats(shardRouting, new ShardPath(false, path, path, shardId), stats, null, null)); + shardStats.add(new ShardStats(shardRouting, new ShardPath(false, path, path, shardId), stats, null, null, null)); } } return IndicesStatsTests.newIndicesStatsResponse( diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java index 66b41d40943d0..be2eb529d86be 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java @@ -47,10 +47,10 @@ public void setUp() throws Exception { super.setUp(); indicesStats = Collections.singletonList(new IndexStats("index-0", "dcvO5uZATE-EhIKc3tk9Bg", new ShardStats[] { // Primaries - new ShardStats(mockShardRouting(true), mockShardPath(), mockCommonStats(), null, null), - new ShardStats(mockShardRouting(true), mockShardPath(), mockCommonStats(), null, null), + new ShardStats(mockShardRouting(true), mockShardPath(), mockCommonStats(), null, null, null), + new ShardStats(mockShardRouting(true), mockShardPath(), mockCommonStats(), null, null, null), // Replica - new ShardStats(mockShardRouting(false), mockShardPath(), mockCommonStats(), null, null) + new ShardStats(mockShardRouting(false), mockShardPath(), mockCommonStats(), null, null, null) })); } From a070b8acc004740a001bd3b01e6fc6417a9e7919 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Wed, 30 Jan 2019 19:21:09 +0100 Subject: [PATCH 062/100] Extract TransportRequestDeduplication from ShardStateAction (#37870) * Extracted the logic for master request duplication so it can be reused by the snapshotting logic * Removed custom listener used by `ShardStateAction` to not leak these into future users of this class * Changed semantics slightly to get rid of redundant instantiations of the composite listener * Relates #37686 --- .../TransportReplicationAction.java | 6 +- .../action/shard/ShardStateAction.java | 145 ++---------------- .../cluster/IndicesClusterStateService.java | 3 +- .../TransportRequestDeduplicator.java | 114 ++++++++++++++ .../action/shard/ShardStateActionTests.java | 81 ++-------- .../discovery/ClusterDisruptionIT.java | 7 +- .../TransportRequestDeduplicatorTests.java | 91 +++++++++++ 7 files changed, 241 insertions(+), 206 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/transport/TransportRequestDeduplicator.java create mode 100644 server/src/test/java/org/elasticsearch/transport/TransportRequestDeduplicatorTests.java diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java index 4894ca1f772c4..c0f0278479a0c 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java @@ -1192,12 +1192,12 @@ public void markShardCopyAsStaleIfNeeded(ShardId shardId, String allocationId, R onSuccess.run(); } - protected final ShardStateAction.Listener createShardActionListener(final Runnable onSuccess, + protected final ActionListener createShardActionListener(final Runnable onSuccess, final Consumer onPrimaryDemoted, final Consumer onIgnoredFailure) { - return new ShardStateAction.Listener() { + return new ActionListener() { @Override - public void onSuccess() { + public void onResponse(Void aVoid) { onSuccess.run(); } diff --git a/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java b/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java index 4419d921a3b4a..071885202c458 100644 --- a/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java +++ b/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java @@ -25,6 +25,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateObserver; @@ -48,18 +49,17 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.node.NodeClosedException; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.ConnectTransportException; import org.elasticsearch.transport.EmptyTransportResponseHandler; -import org.elasticsearch.transport.NodeDisconnectedException; import org.elasticsearch.transport.RemoteTransportException; import org.elasticsearch.transport.TransportChannel; import org.elasticsearch.transport.TransportException; import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.transport.TransportRequestDeduplicator; import org.elasticsearch.transport.TransportRequestHandler; import org.elasticsearch.transport.TransportResponse; import org.elasticsearch.transport.TransportService; @@ -71,7 +71,6 @@ import java.util.Locale; import java.util.Objects; import java.util.Set; -import java.util.concurrent.ConcurrentMap; import java.util.function.Predicate; import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM; @@ -89,7 +88,7 @@ public class ShardStateAction { // a list of shards that failed during replication // we keep track of these shards in order to avoid sending duplicate failed shard requests for a single failing shard. - private final ConcurrentMap remoteFailedShardsCache = ConcurrentCollections.newConcurrentMap(); + private final TransportRequestDeduplicator remoteFailedShardsDeduplicator = new TransportRequestDeduplicator<>(); @Inject public ShardStateAction(ClusterService clusterService, TransportService transportService, @@ -106,7 +105,7 @@ public ShardStateAction(ClusterService clusterService, TransportService transpor } private void sendShardAction(final String actionName, final ClusterState currentState, - final TransportRequest request, final Listener listener) { + final TransportRequest request, final ActionListener listener) { ClusterStateObserver observer = new ClusterStateObserver(currentState, clusterService, null, logger, threadPool.getThreadContext()); DiscoveryNode masterNode = currentState.nodes().getMasterNode(); @@ -120,7 +119,7 @@ private void sendShardAction(final String actionName, final ClusterState current actionName, request, new EmptyTransportResponseHandler(ThreadPool.Names.SAME) { @Override public void handleResponse(TransportResponse.Empty response) { - listener.onSuccess(); + listener.onResponse(null); } @Override @@ -163,44 +162,22 @@ private static boolean isMasterChannelException(TransportException exp) { * @param listener callback upon completion of the request */ public void remoteShardFailed(final ShardId shardId, String allocationId, long primaryTerm, boolean markAsStale, final String message, - @Nullable final Exception failure, Listener listener) { + @Nullable final Exception failure, ActionListener listener) { assert primaryTerm > 0L : "primary term should be strictly positive"; - final FailedShardEntry shardEntry = new FailedShardEntry(shardId, allocationId, primaryTerm, message, failure, markAsStale); - final CompositeListener compositeListener = new CompositeListener(listener); - final CompositeListener existingListener = remoteFailedShardsCache.putIfAbsent(shardEntry, compositeListener); - if (existingListener == null) { - sendShardAction(SHARD_FAILED_ACTION_NAME, clusterService.state(), shardEntry, new Listener() { - @Override - public void onSuccess() { - try { - compositeListener.onSuccess(); - } finally { - remoteFailedShardsCache.remove(shardEntry); - } - } - @Override - public void onFailure(Exception e) { - try { - compositeListener.onFailure(e); - } finally { - remoteFailedShardsCache.remove(shardEntry); - } - } - }); - } else { - existingListener.addListener(listener); - } + remoteFailedShardsDeduplicator.executeOnce( + new FailedShardEntry(shardId, allocationId, primaryTerm, message, failure, markAsStale), listener, + (req, reqListener) -> sendShardAction(SHARD_FAILED_ACTION_NAME, clusterService.state(), req, reqListener)); } int remoteShardFailedCacheSize() { - return remoteFailedShardsCache.size(); + return remoteFailedShardsDeduplicator.size(); } /** * Send a shard failed request to the master node to update the cluster state when a shard on the local node failed. */ public void localShardFailed(final ShardRouting shardRouting, final String message, - @Nullable final Exception failure, Listener listener) { + @Nullable final Exception failure, ActionListener listener) { localShardFailed(shardRouting, message, failure, listener, clusterService.state()); } @@ -208,7 +185,7 @@ public void localShardFailed(final ShardRouting shardRouting, final String messa * Send a shard failed request to the master node to update the cluster state when a shard on the local node failed. */ public void localShardFailed(final ShardRouting shardRouting, final String message, @Nullable final Exception failure, - Listener listener, final ClusterState currentState) { + ActionListener listener, final ClusterState currentState) { FailedShardEntry shardEntry = new FailedShardEntry(shardRouting.shardId(), shardRouting.allocationId().getId(), 0L, message, failure, true); sendShardAction(SHARD_FAILED_ACTION_NAME, currentState, shardEntry, listener); @@ -216,7 +193,8 @@ public void localShardFailed(final ShardRouting shardRouting, final String messa // visible for testing protected void waitForNewMasterAndRetry(String actionName, ClusterStateObserver observer, - TransportRequest request, Listener listener, Predicate changePredicate) { + TransportRequest request, ActionListener listener, + Predicate changePredicate) { observer.waitForNextChange(new ClusterStateObserver.Listener() { @Override public void onNewClusterState(ClusterState state) { @@ -497,14 +475,14 @@ public int hashCode() { public void shardStarted(final ShardRouting shardRouting, final long primaryTerm, final String message, - final Listener listener) { + final ActionListener listener) { shardStarted(shardRouting, primaryTerm, message, listener, clusterService.state()); } public void shardStarted(final ShardRouting shardRouting, final long primaryTerm, final String message, - final Listener listener, + final ActionListener listener, final ClusterState currentState) { StartedShardEntry entry = new StartedShardEntry(shardRouting.shardId(), shardRouting.allocationId().getId(), primaryTerm, message); sendShardAction(SHARD_STARTED_ACTION_NAME, currentState, entry, listener); @@ -670,97 +648,6 @@ public String toString() { } } - public interface Listener { - - default void onSuccess() { - } - - /** - * Notification for non-channel exceptions that are not handled - * by {@link ShardStateAction}. - * - * The exceptions that are handled by {@link ShardStateAction} - * are: - * - {@link NotMasterException} - * - {@link NodeDisconnectedException} - * - {@link FailedToCommitClusterStateException} - * - * Any other exception is communicated to the requester via - * this notification. - * - * @param e the unexpected cause of the failure on the master - */ - default void onFailure(final Exception e) { - } - - } - - /** - * A composite listener that allows registering multiple listeners dynamically. - */ - static final class CompositeListener implements Listener { - private boolean isNotified = false; - private Exception failure = null; - private final List listeners = new ArrayList<>(); - - CompositeListener(Listener listener) { - listeners.add(listener); - } - - void addListener(Listener listener) { - final boolean ready; - synchronized (this) { - ready = this.isNotified; - if (ready == false) { - listeners.add(listener); - } - } - if (ready) { - if (failure != null) { - listener.onFailure(failure); - } else { - listener.onSuccess(); - } - } - } - - private void onCompleted(Exception failure) { - synchronized (this) { - this.failure = failure; - this.isNotified = true; - } - RuntimeException firstException = null; - for (Listener listener : listeners) { - try { - if (failure != null) { - listener.onFailure(failure); - } else { - listener.onSuccess(); - } - } catch (RuntimeException innerEx) { - if (firstException == null) { - firstException = innerEx; - } else { - firstException.addSuppressed(innerEx); - } - } - } - if (firstException != null) { - throw firstException; - } - } - - @Override - public void onSuccess() { - onCompleted(null); - } - - @Override - public void onFailure(Exception failure) { - onCompleted(failure); - } - } - public static class NoLongerPrimaryShardException extends ElasticsearchException { public NoLongerPrimaryShardException(ShardId shardId, String msg) { diff --git a/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java index 5955a749fea34..57ec87d1c6493 100644 --- a/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java +++ b/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java @@ -109,8 +109,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple private final ShardStateAction shardStateAction; private final NodeMappingRefreshAction nodeMappingRefreshAction; - private static final ShardStateAction.Listener SHARD_STATE_ACTION_LISTENER = new ShardStateAction.Listener() { - }; + private static final ActionListener SHARD_STATE_ACTION_LISTENER = ActionListener.wrap(() -> {}); private final Settings settings; // a list of shards that failed during recovery diff --git a/server/src/main/java/org/elasticsearch/transport/TransportRequestDeduplicator.java b/server/src/main/java/org/elasticsearch/transport/TransportRequestDeduplicator.java new file mode 100644 index 0000000000000..d929ef34ce2c3 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/transport/TransportRequestDeduplicator.java @@ -0,0 +1,114 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.transport; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ConcurrentMap; +import java.util.function.BiConsumer; + +/** + * Deduplicator for {@link TransportRequest}s that keeps track of {@link TransportRequest}s that should + * not be sent in parallel. + * @param Transport Request Class + */ +public final class TransportRequestDeduplicator { + + private final ConcurrentMap requests = ConcurrentCollections.newConcurrentMap(); + + /** + * Ensures a given request not executed multiple times when another equal request is already in-flight. + * If the request is not yet known to the deduplicator it will invoke the passed callback with an {@link ActionListener} + * that must be completed by the caller when the request completes. Once that listener is completed the request will be removed from + * the deduplicator's internal state. If the request is already known to the deduplicator it will keep + * track of the given listener and invoke it when the listener passed to the callback on first invocation is completed. + * @param request Request to deduplicate + * @param listener Listener to invoke on request completion + * @param callback Callback to be invoked with request and completion listener the first time the request is added to the deduplicator + */ + public void executeOnce(T request, ActionListener listener, BiConsumer> callback) { + ActionListener completionListener = requests.computeIfAbsent(request, CompositeListener::new).addListener(listener); + if (completionListener != null) { + callback.accept(request, completionListener); + } + } + + public int size() { + return requests.size(); + } + + private final class CompositeListener implements ActionListener { + + private final List> listeners = new ArrayList<>(); + + private final T request; + + private boolean isNotified; + private Exception failure; + + CompositeListener(T request) { + this.request = request; + } + + CompositeListener addListener(ActionListener listener) { + synchronized (this) { + if (this.isNotified == false) { + listeners.add(listener); + return listeners.size() == 1 ? this : null; + } + } + if (failure != null) { + listener.onFailure(failure); + } else { + listener.onResponse(null); + } + return null; + } + + private void onCompleted(Exception failure) { + synchronized (this) { + this.failure = failure; + this.isNotified = true; + } + try { + if (failure == null) { + ActionListener.onResponse(listeners, null); + } else { + ActionListener.onFailure(listeners, failure); + } + } finally { + requests.remove(request); + } + } + + @Override + public void onResponse(final Void aVoid) { + onCompleted(null); + } + + @Override + public void onFailure(Exception failure) { + onCompleted(failure); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java b/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java index a800c0c79929c..69743c101ee10 100644 --- a/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java @@ -22,6 +22,7 @@ import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.util.SetOnce; import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.replication.ClusterStateCreationUtils; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateObserver; @@ -75,7 +76,6 @@ import static org.elasticsearch.test.VersionUtils.randomCompatibleVersion; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; -import static org.hamcrest.CoreMatchers.sameInstance; import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; @@ -110,7 +110,7 @@ public void setOnAfterWaitForNewMasterAndRetry(Runnable onAfterWaitForNewMasterA @Override protected void waitForNewMasterAndRetry(String actionName, ClusterStateObserver observer, TransportRequest request, - Listener listener, Predicate changePredicate) { + ActionListener listener, Predicate changePredicate) { onBeforeWaitForNewMasterAndRetry.run(); super.waitForNewMasterAndRetry(actionName, observer, request, listener, changePredicate); onAfterWaitForNewMasterAndRetry.run(); @@ -197,9 +197,9 @@ public void testNoMaster() throws InterruptedException { }); ShardRouting failedShard = getRandomShardRouting(index); - shardStateAction.localShardFailed(failedShard, "test", getSimulatedFailure(), new ShardStateAction.Listener() { + shardStateAction.localShardFailed(failedShard, "test", getSimulatedFailure(), new ActionListener() { @Override - public void onSuccess() { + public void onResponse(Void aVoid) { success.set(true); latch.countDown(); } @@ -246,9 +246,9 @@ public void testMasterChannelException() throws InterruptedException { setUpMasterRetryVerification(numberOfRetries, retries, latch, retryLoop); ShardRouting failedShard = getRandomShardRouting(index); - shardStateAction.localShardFailed(failedShard, "test", getSimulatedFailure(), new ShardStateAction.Listener() { + shardStateAction.localShardFailed(failedShard, "test", getSimulatedFailure(), new ActionListener() { @Override - public void onSuccess() { + public void onResponse(Void aVoid) { success.set(true); latch.countDown(); } @@ -343,9 +343,9 @@ public void testCacheRemoteShardFailed() throws Exception { long primaryTerm = randomLongBetween(1, Long.MAX_VALUE); for (int i = 0; i < numListeners; i++) { shardStateAction.remoteShardFailed(failedShard.shardId(), failedShard.allocationId().getId(), - primaryTerm, markAsStale, "test", getSimulatedFailure(), new ShardStateAction.Listener() { + primaryTerm, markAsStale, "test", getSimulatedFailure(), new ActionListener() { @Override - public void onSuccess() { + public void onResponse(Void aVoid) { latch.countDown(); } @Override @@ -394,9 +394,9 @@ public void testRemoteShardFailedConcurrently() throws Exception { ShardRouting failedShard = randomFrom(failedShards); shardStateAction.remoteShardFailed(failedShard.shardId(), failedShard.allocationId().getId(), randomLongBetween(1, Long.MAX_VALUE), randomBoolean(), "test", getSimulatedFailure(), - new ShardStateAction.Listener() { + new ActionListener() { @Override - public void onSuccess() { + public void onResponse(Void aVoid) { notifiedResponses.incrementAndGet(); } @Override @@ -561,70 +561,13 @@ BytesReference serialize(Writeable writeable, Version version) throws IOExceptio } } - public void testCompositeListener() throws Exception { - AtomicInteger successCount = new AtomicInteger(); - AtomicInteger failureCount = new AtomicInteger(); - Exception failure = randomBoolean() ? getSimulatedFailure() : null; - ShardStateAction.CompositeListener compositeListener = new ShardStateAction.CompositeListener(new ShardStateAction.Listener() { - @Override - public void onSuccess() { - successCount.incrementAndGet(); - } - @Override - public void onFailure(Exception e) { - assertThat(e, sameInstance(failure)); - failureCount.incrementAndGet(); - } - }); - int iterationsPerThread = scaledRandomIntBetween(100, 1000); - Thread[] threads = new Thread[between(1, 4)]; - Phaser barrier = new Phaser(threads.length + 1); - for (int i = 0; i < threads.length; i++) { - threads[i] = new Thread(() -> { - barrier.arriveAndAwaitAdvance(); - for (int n = 0; n < iterationsPerThread; n++) { - compositeListener.addListener(new ShardStateAction.Listener() { - @Override - public void onSuccess() { - successCount.incrementAndGet(); - } - @Override - public void onFailure(Exception e) { - assertThat(e, sameInstance(failure)); - failureCount.incrementAndGet(); - } - }); - } - }); - threads[i].start(); - } - barrier.arriveAndAwaitAdvance(); - if (failure != null) { - compositeListener.onFailure(failure); - } else { - compositeListener.onSuccess(); - } - for (Thread t : threads) { - t.join(); - } - assertBusy(() -> { - if (failure != null) { - assertThat(successCount.get(), equalTo(0)); - assertThat(failureCount.get(), equalTo(threads.length*iterationsPerThread + 1)); - } else { - assertThat(successCount.get(), equalTo(threads.length*iterationsPerThread + 1)); - assertThat(failureCount.get(), equalTo(0)); - } - }); - } - - private static class TestListener implements ShardStateAction.Listener { + private static class TestListener implements ActionListener { private final SetOnce failure = new SetOnce<>(); private final CountDownLatch latch = new CountDownLatch(1); @Override - public void onSuccess() { + public void onResponse(Void aVoid) { try { failure.set(null); } finally { diff --git a/server/src/test/java/org/elasticsearch/discovery/ClusterDisruptionIT.java b/server/src/test/java/org/elasticsearch/discovery/ClusterDisruptionIT.java index 330c73b9c02c5..0a9016c20111b 100644 --- a/server/src/test/java/org/elasticsearch/discovery/ClusterDisruptionIT.java +++ b/server/src/test/java/org/elasticsearch/discovery/ClusterDisruptionIT.java @@ -22,6 +22,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.index.CorruptIndexException; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.NoShardAvailableActionException; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexResponse; @@ -317,10 +318,10 @@ public void testSendingShardFailure() throws Exception { setDisruptionScheme(networkDisruption); networkDisruption.startDisrupting(); - service.localShardFailed(failedShard, "simulated", new CorruptIndexException("simulated", (String) null), new - ShardStateAction.Listener() { + service.localShardFailed(failedShard, "simulated", new CorruptIndexException("simulated", (String) null), + new ActionListener() { @Override - public void onSuccess() { + public void onResponse(final Void aVoid) { success.set(true); latch.countDown(); } diff --git a/server/src/test/java/org/elasticsearch/transport/TransportRequestDeduplicatorTests.java b/server/src/test/java/org/elasticsearch/transport/TransportRequestDeduplicatorTests.java new file mode 100644 index 0000000000000..ab178134995a7 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/transport/TransportRequestDeduplicatorTests.java @@ -0,0 +1,91 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.transport; + +import org.apache.lucene.util.SetOnce; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.test.ESTestCase; + +import java.util.concurrent.Phaser; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.sameInstance; + +public class TransportRequestDeduplicatorTests extends ESTestCase { + + public void testRequestDeduplication() throws Exception { + AtomicInteger successCount = new AtomicInteger(); + AtomicInteger failureCount = new AtomicInteger(); + Exception failure = randomBoolean() ? new TransportException("simulated") : null; + final TransportRequest request = new TransportRequest() { + @Override + public void setParentTask(final TaskId taskId) { + } + }; + final TransportRequestDeduplicator deduplicator = new TransportRequestDeduplicator<>(); + final SetOnce> listenerHolder = new SetOnce<>(); + int iterationsPerThread = scaledRandomIntBetween(100, 1000); + Thread[] threads = new Thread[between(1, 4)]; + Phaser barrier = new Phaser(threads.length + 1); + for (int i = 0; i < threads.length; i++) { + threads[i] = new Thread(() -> { + barrier.arriveAndAwaitAdvance(); + for (int n = 0; n < iterationsPerThread; n++) { + deduplicator.executeOnce(request, new ActionListener() { + @Override + public void onResponse(Void aVoid) { + successCount.incrementAndGet(); + } + + @Override + public void onFailure(Exception e) { + assertThat(e, sameInstance(failure)); + failureCount.incrementAndGet(); + } + }, (req, reqListener) -> listenerHolder.set(reqListener)); + } + }); + threads[i].start(); + } + barrier.arriveAndAwaitAdvance(); + for (Thread t : threads) { + t.join(); + } + final ActionListener listener = listenerHolder.get(); + assertThat(deduplicator.size(), equalTo(1)); + if (failure != null) { + listener.onFailure(failure); + } else { + listener.onResponse(null); + } + assertThat(deduplicator.size(), equalTo(0)); + assertBusy(() -> { + if (failure != null) { + assertThat(successCount.get(), equalTo(0)); + assertThat(failureCount.get(), equalTo(threads.length * iterationsPerThread)); + } else { + assertThat(successCount.get(), equalTo(threads.length * iterationsPerThread)); + assertThat(failureCount.get(), equalTo(0)); + } + }); + } + +} From cac6b8e06f051d68919faf6081f1c87fa5b6757d Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Wed, 30 Jan 2019 11:24:18 -0700 Subject: [PATCH 063/100] Add ECS schema for user-agent ingest processor (#37727) (#37984) * Add ECS schema for user-agent ingest processor (#37727) This switches the format of the user agent processor to use the schema from [ECS](https://github.com/elastic/ecs). So rather than something like this: ``` { "patch" : "3538", "major" : "70", "minor" : "0", "os" : "Mac OS X 10.14.1", "os_minor" : "14", "os_major" : "10", "name" : "Chrome", "os_name" : "Mac OS X", "device" : "Other" } ``` The structure is now like this: ``` { "name" : "Chrome", "original" : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36", "os" : { "name" : "Mac OS X", "version" : "10.14.1", "full" : "Mac OS X 10.14.1" }, "device" : "Other", "version" : "70.0.3538.102" } ``` This is now the default for 7.0. The deprecated `ecs` setting in 6.x is not supported. Resolves #37329 * Remove `ecs` setting from docs --- .../ingest/processors/user-agent.asciidoc | 14 +-- .../migration/migrate_7_0/settings.asciidoc | 6 + .../ingest/useragent/UserAgentProcessor.java | 103 ++++++++++-------- .../UserAgentProcessorFactoryTests.java | 4 +- .../useragent/UserAgentProcessorTests.java | 44 +++----- .../20_useragent_processor.yml | 19 +--- .../test/ingest-useragent/30_custom_regex.yml | 9 +- 7 files changed, 92 insertions(+), 107 deletions(-) diff --git a/docs/reference/ingest/processors/user-agent.asciidoc b/docs/reference/ingest/processors/user-agent.asciidoc index 201e3beab8313..f6b6d46fe7b9d 100644 --- a/docs/reference/ingest/processors/user-agent.asciidoc +++ b/docs/reference/ingest/processors/user-agent.asciidoc @@ -60,13 +60,13 @@ Which returns "agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", "user_agent": { "name": "Chrome", - "major": "51", - "minor": "0", - "patch": "2704", - "os_name": "Mac OS X", - "os": "Mac OS X 10.10.5", - "os_major": "10", - "os_minor": "10", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", + "version": "51.0.2704", + "os": { + "name": "Mac OS X", + "version": "10.10.5", + "full": "Mac OS X 10.10.5" + }, "device": "Other" } } diff --git a/docs/reference/migration/migrate_7_0/settings.asciidoc b/docs/reference/migration/migrate_7_0/settings.asciidoc index 6e9f7451e094f..c6874856011ce 100644 --- a/docs/reference/migration/migrate_7_0/settings.asciidoc +++ b/docs/reference/migration/migrate_7_0/settings.asciidoc @@ -182,3 +182,9 @@ could have lead to dropping audit events while the operations on the system were allowed to continue as usual. The recommended replacement is the use of the `logfile` audit output type and using other components from the Elastic Stack to handle the indexing part. + +[float] +[[ingest-user-agent-ecs-always]] +==== Ingest User Agent processor always uses `ecs` output format +The deprecated `ecs` setting for the user agent ingest processor has been +removed. https://github.com/elastic/ecs[ECS] format is now the default. diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java index 6e7f588f0bd8a..6f2518eede673 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java @@ -19,6 +19,8 @@ package org.elasticsearch.ingest.useragent; +import org.apache.logging.log4j.LogManager; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; @@ -40,6 +42,8 @@ public class UserAgentProcessor extends AbstractProcessor { + private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(UserAgentProcessor.class)); + public static final String TYPE = "user_agent"; private final String field; @@ -63,7 +67,7 @@ boolean isIgnoreMissing() { } @Override - public IngestDocument execute(IngestDocument ingestDocument) throws Exception { + public IngestDocument execute(IngestDocument ingestDocument) { String userAgent = ingestDocument.getFieldValue(field, String.class, ignoreMissing); if (userAgent == null && ignoreMissing) { @@ -75,68 +79,64 @@ public IngestDocument execute(IngestDocument ingestDocument) throws Exception { Details uaClient = parser.parse(userAgent); Map uaDetails = new HashMap<>(); + + // Parse the user agent in the ECS (Elastic Common Schema) format for (Property property : this.properties) { switch (property) { + case ORIGINAL: + uaDetails.put("original", userAgent); + break; case NAME: if (uaClient.userAgent != null && uaClient.userAgent.name != null) { uaDetails.put("name", uaClient.userAgent.name); - } - else { + } else { uaDetails.put("name", "Other"); } break; - case MAJOR: + case VERSION: + StringBuilder version = new StringBuilder(); if (uaClient.userAgent != null && uaClient.userAgent.major != null) { - uaDetails.put("major", uaClient.userAgent.major); - } - break; - case MINOR: - if (uaClient.userAgent != null && uaClient.userAgent.minor != null) { - uaDetails.put("minor", uaClient.userAgent.minor); - } - break; - case PATCH: - if (uaClient.userAgent != null && uaClient.userAgent.patch != null) { - uaDetails.put("patch", uaClient.userAgent.patch); - } - break; - case BUILD: - if (uaClient.userAgent != null && uaClient.userAgent.build != null) { - uaDetails.put("build", uaClient.userAgent.build); + version.append(uaClient.userAgent.major); + if (uaClient.userAgent.minor != null) { + version.append(".").append(uaClient.userAgent.minor); + if (uaClient.userAgent.patch != null) { + version.append(".").append(uaClient.userAgent.patch); + if (uaClient.userAgent.build != null) { + version.append(".").append(uaClient.userAgent.build); + } + } + } + uaDetails.put("version", version.toString()); } break; case OS: if (uaClient.operatingSystem != null) { - uaDetails.put("os", buildFullOSName(uaClient.operatingSystem)); - } - else { - uaDetails.put("os", "Other"); - } - - break; - case OS_NAME: - if (uaClient.operatingSystem != null && uaClient.operatingSystem.name != null) { - uaDetails.put("os_name", uaClient.operatingSystem.name); - } - else { - uaDetails.put("os_name", "Other"); - } - break; - case OS_MAJOR: - if (uaClient.operatingSystem != null && uaClient.operatingSystem.major != null) { - uaDetails.put("os_major", uaClient.operatingSystem.major); - } - break; - case OS_MINOR: - if (uaClient.operatingSystem != null && uaClient.operatingSystem.minor != null) { - uaDetails.put("os_minor", uaClient.operatingSystem.minor); + Map osDetails = new HashMap<>(3); + if (uaClient.operatingSystem.name != null) { + osDetails.put("name", uaClient.operatingSystem.name); + StringBuilder sb = new StringBuilder(); + if (uaClient.operatingSystem.major != null) { + sb.append(uaClient.operatingSystem.major); + if (uaClient.operatingSystem.minor != null) { + sb.append(".").append(uaClient.operatingSystem.minor); + if (uaClient.operatingSystem.patch != null) { + sb.append(".").append(uaClient.operatingSystem.patch); + if (uaClient.operatingSystem.build != null) { + sb.append(".").append(uaClient.operatingSystem.build); + } + } + } + osDetails.put("version", sb.toString()); + osDetails.put("full", uaClient.operatingSystem.name + " " + sb.toString()); + } + uaDetails.put("os", osDetails); + } } break; case DEVICE: if (uaClient.device != null && uaClient.device.name != null) { uaDetails.put("device", uaClient.device.name); - } - else { + } else { uaDetails.put("device", "Other"); } break; @@ -215,6 +215,10 @@ public UserAgentProcessor create(Map factories, Strin String regexFilename = readStringProperty(TYPE, processorTag, config, "regex_file", IngestUserAgentPlugin.DEFAULT_PARSER_NAME); List propertyNames = readOptionalList(TYPE, processorTag, config, "properties"); boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false); + Object ecsValue = config.remove("ecs"); + if (ecsValue != null) { + deprecationLogger.deprecated("setting [ecs] is deprecated as ECS format is the default and only option"); + } UserAgentParser parser = userAgentParsers.get(regexFilename); if (parser == null) { @@ -242,13 +246,16 @@ public UserAgentProcessor create(Map factories, Strin enum Property { - NAME, MAJOR, MINOR, PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD; + NAME, + OS, + DEVICE, + ORIGINAL, + VERSION; public static Property parseProperty(String propertyName) { try { return valueOf(propertyName.toUpperCase(Locale.ROOT)); - } - catch (IllegalArgumentException e) { + } catch (IllegalArgumentException e) { throw new IllegalArgumentException("illegal property value [" + propertyName + "]. valid values are " + Arrays.toString(EnumSet.allOf(Property.class).toArray())); } diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java index d9c6fc17620da..f723c13f23022 100644 --- a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java @@ -178,8 +178,8 @@ public void testInvalidProperty() throws Exception { config.put("properties", Collections.singletonList("invalid")); ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config)); - assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, MAJOR, MINOR, " - + "PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD]")); + assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, OS, DEVICE, " + + "ORIGINAL, VERSION]")); } public void testInvalidPropertiesType() throws Exception { diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java index 0a8b453724c90..3938fccd832a3 100644 --- a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java @@ -103,16 +103,13 @@ public void testCommonBrowser() throws Exception { Map target = (Map) data.get("target_field"); assertThat(target.get("name"), is("Chrome")); - assertThat(target.get("major"), is("33")); - assertThat(target.get("minor"), is("0")); - assertThat(target.get("patch"), is("1750")); - assertNull(target.get("build")); - - assertThat(target.get("os"), is("Mac OS X 10.9.2")); - assertThat(target.get("os_name"), is("Mac OS X")); - assertThat(target.get("os_major"), is("10")); - assertThat(target.get("os_minor"), is("9")); + assertThat(target.get("version"), is("33.0.1750")); + Map os = new HashMap<>(); + os.put("name", "Mac OS X"); + os.put("version", "10.9.2"); + os.put("full", "Mac OS X 10.9.2"); + assertThat(target.get("os"), is(os)); assertThat(target.get("device"), is("Other")); } @@ -131,15 +128,13 @@ public void testUncommonDevice() throws Exception { Map target = (Map) data.get("target_field"); assertThat(target.get("name"), is("Android")); - assertThat(target.get("major"), is("3")); - assertThat(target.get("minor"), is("0")); - assertNull(target.get("patch")); - assertNull(target.get("build")); + assertThat(target.get("version"), is("3.0")); - assertThat(target.get("os"), is("Android 3.0")); - assertThat(target.get("os_name"), is("Android")); - assertThat(target.get("os_major"), is("3")); - assertThat(target.get("os_minor"), is("0")); + Map os = new HashMap<>(); + os.put("name", "Android"); + os.put("version", "3.0"); + os.put("full", "Android 3.0"); + assertThat(target.get("os"), is(os)); assertThat(target.get("device"), is("Motorola Xoom")); } @@ -158,15 +153,9 @@ public void testSpider() throws Exception { Map target = (Map) data.get("target_field"); assertThat(target.get("name"), is("EasouSpider")); - assertNull(target.get("major")); - assertNull(target.get("minor")); - assertNull(target.get("patch")); - assertNull(target.get("build")); - assertThat(target.get("os"), is("Other")); - assertThat(target.get("os_name"), is("Other")); - assertNull(target.get("os_major")); - assertNull(target.get("os_minor")); + assertNull(target.get("version")); + assertNull(target.get("os")); assertThat(target.get("device"), is("Spider")); } @@ -190,10 +179,7 @@ public void testUnknown() throws Exception { assertNull(target.get("patch")); assertNull(target.get("build")); - assertThat(target.get("os"), is("Other")); - assertThat(target.get("os_name"), is("Other")); - assertNull(target.get("os_major")); - assertNull(target.get("os_minor")); + assertNull(target.get("os")); assertThat(target.get("device"), is("Other")); } diff --git a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml b/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml index 28c218edd6935..fc44d7261e80f 100644 --- a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml +++ b/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml @@ -29,13 +29,9 @@ id: 1 - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } - match: { _source.user_agent.name: "Chrome" } - - match: { _source.user_agent.os: "Mac OS X 10.9.2" } - - match: { _source.user_agent.os_name: "Mac OS X" } - - match: { _source.user_agent.os_major: "10" } - - match: { _source.user_agent.os_minor: "9" } - - match: { _source.user_agent.major: "33" } - - match: { _source.user_agent.minor: "0" } - - match: { _source.user_agent.patch: "1750" } + - match: { _source.user_agent.original: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } + - match: { _source.user_agent.os: {"name":"Mac OS X", "version":"10.9.2", "full":"Mac OS X 10.9.2"} } + - match: { _source.user_agent.version: "33.0.1750" } - match: { _source.user_agent.device: "Other" } --- @@ -70,13 +66,8 @@ index: test id: 1 - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } - - match: { _source.field2.os: "Mac OS X 10.9.2" } + - match: { _source.field2.os.full: "Mac OS X 10.9.2" } - is_false: _source.user_agent - is_false: _source.field2.name - - is_false: _source.field2.os_name - - is_false: _source.field2.os_major - - is_false: _source.field2.os_minor - - is_false: _source.field2.major - - is_false: _source.field2.minor - - is_false: _source.field2.patch - is_false: _source.field2.device + - is_false: _source.field2.original diff --git a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml b/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml index 22df584e13166..ac90a3457fa65 100644 --- a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml +++ b/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml @@ -30,11 +30,6 @@ id: 1 - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } - match: { _source.user_agent.name: "Test" } - - match: { _source.user_agent.os: "Other" } - - match: { _source.user_agent.os_name: "Other" } - match: { _source.user_agent.device: "Other" } - - is_false: _source.user_agent.os_major - - is_false: _source.user_agent.os_minor - - is_false: _source.user_agent.major - - is_false: _source.user_agent.minor - - is_false: _source.user_agent.patch + - is_false: _source.user_agent.os + - is_false: _source.user_agent.version From 5433af28e33daeda6d30e8cd41ac1e69a548334c Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 30 Jan 2019 19:33:00 +0100 Subject: [PATCH 064/100] Fixed test bug, lastFollowTime is null if there are no follower indices. --- .../org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java index b95e8fc7c4008..f2fb475816ec2 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java @@ -27,6 +27,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.nullValue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -36,7 +37,7 @@ public class CCRFeatureSetTests extends ESTestCase { private ClusterService clusterService; @Before - public void init() throws Exception { + public void init() { licenseState = mock(XPackLicenseState.class); clusterService = mock(ClusterService.class); } @@ -116,7 +117,11 @@ public void testUsageStats() throws Exception { assertThat(ccrUsage.available(), equalTo(ccrFeatureSet.available())); assertThat(ccrUsage.getNumberOfFollowerIndices(), equalTo(numFollowerIndices)); - assertThat(ccrUsage.getLastFollowTimeInMillis(), greaterThanOrEqualTo(0L)); + if (numFollowerIndices != 0) { + assertThat(ccrUsage.getLastFollowTimeInMillis(), greaterThanOrEqualTo(0L)); + } else { + assertThat(ccrUsage.getLastFollowTimeInMillis(), nullValue()); + } assertThat(ccrUsage.getNumberOfAutoFollowPatterns(), equalTo(numAutoFollowPatterns)); } From 36ee78d92468901be372b5218890f4c3e59f564d Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Wed, 30 Jan 2019 11:01:45 -0800 Subject: [PATCH 065/100] Add test coverage for Painless general casting of boolean and Boolean (#37780) This adds test coverage for general casts in Painless between boolean and other types and Boolean and other types. --- .../painless/StandardCastTests.java | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StandardCastTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StandardCastTests.java index 739d9d021a427..d19ba0847b2de 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StandardCastTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StandardCastTests.java @@ -641,4 +641,158 @@ public void testStringCasts() { expectScriptThrows(ClassCastException.class, () -> exec("String o = 'string'; ArrayList b = o;")); expectScriptThrows(ClassCastException.class, () -> exec("String o = 'string'; ArrayList b = (ArrayList)o;")); } + + public void testPrimitiveBooleanCasts() { + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Object n = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Object n = (Object)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Number n = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Number n = (boolean)o;")); + + exec("boolean o = true; boolean b = o;"); + exec("boolean o = true; boolean b = (boolean)o;"); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; byte b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; byte b = (byte)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; short b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; short b = (short)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; char b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; char b = (char)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; int b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; int b = (int)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; long b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; long b = (long)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; float b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; float b = (float)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; double b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; double b = (double)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Boolean b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Boolean b = (Boolean)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Byte b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Byte b = (Byte)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Short b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Short b = (Short)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Character b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Character b = (Character)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Integer b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Integer b = (Integer)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Long b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Long b = (Long)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Float b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Float b = (Float)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Double b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; Double b = (Double)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; ArrayList b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("boolean o = true; ArrayList b = (ArrayList)o;")); + } + + public void testBoxedBooleanCasts() { + exec("Boolean o = Boolean.valueOf(true); Object n = o;"); + exec("Boolean o = null; Object n = o;"); + exec("Boolean o = Boolean.valueOf(true); Object n = (Object)o;"); + exec("Boolean o = null; Object n = (Object)o;"); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); Number n = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; Number n = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); Number n = (Boolean)o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; Number n = (Boolean)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); boolean b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; boolean b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); boolean b = (boolean)o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; boolean b = (boolean)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); byte b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; byte b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); byte b = (byte)o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; byte b = (byte)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); short b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; short b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); short b = (short)o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; short b = (short)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); char b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; char b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); char b = (char)o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; char b = (char)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); int b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; int b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); int b = (int)o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; int b = (int)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); long b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; long b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); long b = (long)o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; long b = (long)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); float b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; float b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); float b = (float)o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; float b = (float)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); double b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; double b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); double b = (double)o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; double b = (double)o;")); + + exec("Boolean o = Boolean.valueOf(true); Boolean b = o;"); + exec("Boolean o = null; Boolean b = o;"); + exec("Boolean o = Boolean.valueOf(true); Boolean b = (Boolean)o;"); + exec("Boolean o = null; Boolean b = (Boolean)o;"); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); Byte b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; Byte b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); Byte b = (Byte)o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; Byte b = (Byte)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); Short b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; Short b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); Short b = (Short)o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; Short b = (Short)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); Character b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; Character b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); Character b = (Character)o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; Character b = (Character)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); Integer b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; Integer b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); Integer b = (Integer)o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; Integer b = (Integer)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); Long b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; Long b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); Long b = (Long)o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; Long b = (Long)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); Float b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; Float b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); Float b = (Float)o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; Float b = (Float)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); Double b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; Double b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); Double b = (Double)o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = null; Double b = (Double)o;")); + + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); ArrayList b = o;")); + expectScriptThrows(ClassCastException.class, () -> exec("Boolean o = Boolean.valueOf(true); ArrayList b = (ArrayList)o;")); + } } From 6a78b6a31cfe0f06f2656672db12f1d5edfb5aaf Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Wed, 30 Jan 2019 13:12:13 -0600 Subject: [PATCH 066/100] Remove types from watcher docs (#38002) Types have been deprecated and this commit removes the documentation for specifying types in the index action, and search input/transform. Relates #37594 #35190 --- x-pack/docs/build.gradle | 3 +-- x-pack/docs/en/rest-api/watcher/ack-watch.asciidoc | 3 +-- x-pack/docs/en/rest-api/watcher/execute-watch.asciidoc | 2 +- x-pack/docs/en/rest-api/watcher/get-watch.asciidoc | 3 +-- x-pack/docs/en/watcher/actions/index.asciidoc | 9 +++------ x-pack/docs/en/watcher/input/search.asciidoc | 4 ---- x-pack/docs/en/watcher/transform/search.asciidoc | 6 ------ 7 files changed, 7 insertions(+), 23 deletions(-) diff --git a/x-pack/docs/build.gradle b/x-pack/docs/build.gradle index de2400c0e85f0..27f815b1637f1 100644 --- a/x-pack/docs/build.gradle +++ b/x-pack/docs/build.gradle @@ -122,8 +122,7 @@ setups['my_inactive_watch'] = ''' "actions": { "test_index": { "index": { - "index": "test", - "doc_type": "test2" + "index": "test" } } } diff --git a/x-pack/docs/en/rest-api/watcher/ack-watch.asciidoc b/x-pack/docs/en/rest-api/watcher/ack-watch.asciidoc index a1704e9acc329..f1e281819eec3 100644 --- a/x-pack/docs/en/rest-api/watcher/ack-watch.asciidoc +++ b/x-pack/docs/en/rest-api/watcher/ack-watch.asciidoc @@ -68,8 +68,7 @@ PUT _watcher/watch/my_watch "test_index": { "throttle_period": "15m", "index": { - "index": "test", - "doc_type": "test2" + "index": "test" } } } diff --git a/x-pack/docs/en/rest-api/watcher/execute-watch.asciidoc b/x-pack/docs/en/rest-api/watcher/execute-watch.asciidoc index ca3a0de0af781..8c7f747969373 100644 --- a/x-pack/docs/en/rest-api/watcher/execute-watch.asciidoc +++ b/x-pack/docs/en/rest-api/watcher/execute-watch.asciidoc @@ -255,7 +255,7 @@ This is an example of the output: "index": { "response": { "index": "test", - "type": "test2", + "type": "_doc", "version": 1, "created": true, "result": "created", diff --git a/x-pack/docs/en/rest-api/watcher/get-watch.asciidoc b/x-pack/docs/en/rest-api/watcher/get-watch.asciidoc index 7d62b5c76c41d..87a13d0829f9b 100644 --- a/x-pack/docs/en/rest-api/watcher/get-watch.asciidoc +++ b/x-pack/docs/en/rest-api/watcher/get-watch.asciidoc @@ -81,8 +81,7 @@ Response: "actions": { "test_index": { "index": { - "index": "test", - "doc_type": "test2" + "index": "test" } } } diff --git a/x-pack/docs/en/watcher/actions/index.asciidoc b/x-pack/docs/en/watcher/actions/index.asciidoc index 8a31b150f22cb..34fdad7c50d0b 100644 --- a/x-pack/docs/en/watcher/actions/index.asciidoc +++ b/x-pack/docs/en/watcher/actions/index.asciidoc @@ -16,8 +16,7 @@ The following snippet shows a simple `index` action definition: "transform": { ... }, <3> "index" : { "index" : "my-index", <4> - "doc_type" : "my-type", <5> - "doc_id": "my-id" <6> + "doc_id": "my-id" <5> } } } @@ -27,8 +26,7 @@ The following snippet shows a simple `index` action definition: <2> An optional <> to restrict action execution <3> An optional <> to transform the payload and prepare the data that should be indexed <4> The elasticsearch index to store the data to -<5> The document type to store the data as -<6> An optional `_id` for the document, if it should always be the same document. +<5> An optional `_id` for the document, if it should always be the same document. [[index-action-attributes]] @@ -40,7 +38,6 @@ The following snippet shows a simple `index` action definition: | `index` | yes | - | The Elasticsearch index to index into. -| `doc_type` | yes | - | The type of the document the data will be indexed as. | `doc_id` | no | - | The optional `_id` of the document. @@ -75,5 +72,5 @@ When a `_doc` field exists, if the field holds an object, it is extracted and in as a single document. If the field holds an array of objects, each object is treated as a document and the index action indexes all of them in a bulk. -An `_index`, `_type` or `_id` value can be added per document to dynamically set the ID +An `_index`, or `_id` value can be added per document to dynamically set the ID of the indexed document. diff --git a/x-pack/docs/en/watcher/input/search.asciidoc b/x-pack/docs/en/watcher/input/search.asciidoc index 3b50b22081b60..d4548a159a640 100644 --- a/x-pack/docs/en/watcher/input/search.asciidoc +++ b/x-pack/docs/en/watcher/input/search.asciidoc @@ -24,7 +24,6 @@ documents from the `logs` index: "search" : { "request" : { "indices" : [ "logs" ], - "types" : [ "event" ], "body" : { "query" : { "match_all" : {}} } @@ -172,9 +171,6 @@ accurately. | `request.indices` | no | - | The indices to search. If omitted, all indices are searched, which is the default behaviour in Elasticsearch. -| `request.types` | no | - | The document types to search for. If omitted, all document types are are - searched, which is the default behaviour in Elasticsearch. - | `request.body` | no | - | The body of the request. The {ref}/search-request-body.html[request body] follows the same structure you normally send in the body of a REST `_search` request. The body can be static text or include `mustache` <>. diff --git a/x-pack/docs/en/watcher/transform/search.asciidoc b/x-pack/docs/en/watcher/transform/search.asciidoc index 56f9304d986ce..d7f468f183182 100644 --- a/x-pack/docs/en/watcher/transform/search.asciidoc +++ b/x-pack/docs/en/watcher/transform/search.asciidoc @@ -56,10 +56,6 @@ The following table lists all available settings for the search transform: | `request.indices` | no | all indices | One or more indices to search on. -| `request.types` | no | all types | One or more document types to search on (may be a - comma-delimited string or an array of document types - names) - | `request.body` | no | `match_all` query | The body of the request. The {ref}/search-request-body.html[request body] follows the same structure you normally send in the body of @@ -105,7 +101,6 @@ time of the watch: "search" : { "request" : { "indices" : [ "logstash-*" ], - "types" : [ "event" ], "body" : { "size" : 0, "query" : { @@ -145,7 +140,6 @@ The following is an example of using templates that refer to provided parameters "search" : { "request" : { "indices" : [ "logstash-*" ], - "types" : [ "event" ], "template" : { "source" : { "size" : 0, From 81c443c9de81f3cd27b2050c0a85ff29d147cd73 Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 30 Jan 2019 20:09:15 +0000 Subject: [PATCH 067/100] Deprecate minimum_master_nodes (#37868) Today we pass `discovery.zen.minimum_master_nodes` to nodes started up in tests, but for 7.x nodes this setting is not required as it has no effect. This commit removes this setting so that nodes are started with more realistic configurations, and deprecates it. --- .../gradle/test/ClusterFormationTasks.groovy | 13 ++- .../migration/migrate_7_0/discovery.asciidoc | 4 +- .../rest/discovery/Zen2RestApiIT.java | 8 +- .../ec2/Ec2DiscoveryUpdateSettingsTests.java | 14 ++-- .../custom_logging/CustomLoggingConfigIT.java | 2 +- qa/rolling-upgrade/build.gradle | 1 - .../test/cluster.put_settings/10_basic.yml | 16 ++-- .../discovery/zen/ElectMasterService.java | 2 +- .../admin/indices/exists/IndicesExistsIT.java | 4 +- .../cluster/MinimumMasterNodesIT.java | 7 +- .../cluster/SpecificMasterNodesIT.java | 7 -- .../coordination/UnsafeBootstrapMasterIT.java | 26 ++---- .../cluster/coordination/Zen1IT.java | 6 +- .../discovery/MasterDisruptionIT.java | 8 +- .../MinimumMasterNodesInClusterStateIT.java | 66 --------------- .../discovery/zen/ZenDiscoveryUnitTests.java | 6 ++ .../gateway/RecoverAfterNodesIT.java | 31 ++++--- .../elasticsearch/test/ESIntegTestCase.java | 11 --- .../test/InternalTestCluster.java | 70 +++++++++++----- .../test/discovery/TestZenDiscovery.java | 8 +- .../test/test/InternalTestClusterTests.java | 80 ++----------------- .../elasticsearch/license/LicensingTests.java | 15 ++-- ...ServerTransportFilterIntegrationTests.java | 13 ++- 23 files changed, 140 insertions(+), 278 deletions(-) delete mode 100644 server/src/test/java/org/elasticsearch/discovery/zen/MinimumMasterNodesInClusterStateIT.java diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index 2fffd67215581..f32d0d858f81f 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -373,18 +373,15 @@ class ClusterFormationTasks { 'path.repo' : "${node.sharedDir}/repo", 'path.shared_data' : "${node.sharedDir}/", // Define a node attribute so we can test that it exists - 'node.attr.testattr' : 'test' + 'node.attr.testattr' : 'test', + // Don't wait for state, just start up quickly. This will also allow new and old nodes in the BWC case to become the master + 'discovery.initial_state_timeout' : '0s' ] int minimumMasterNodes = node.config.minimumMasterNodes.call() - if (minimumMasterNodes > 0) { + if (node.nodeVersion.before("7.0.0") && minimumMasterNodes > 0) { esConfig['discovery.zen.minimum_master_nodes'] = minimumMasterNodes } - if (minimumMasterNodes > 1) { - // don't wait for state.. just start up quickly - // this will also allow new and old nodes in the BWC case to become the master - esConfig['discovery.initial_state_timeout'] = '0s' - } - if (esConfig.containsKey('discovery.zen.master_election.wait_for_joins_timeout') == false) { + if (node.nodeVersion.before("7.0.0") && esConfig.containsKey('discovery.zen.master_election.wait_for_joins_timeout') == false) { // If a node decides to become master based on partial information from the pinging, don't let it hang for 30 seconds to correct // its mistake. Instead, only wait 5s to do another round of pinging. // This is necessary since we use 30s as the default timeout in REST requests waiting for cluster formation diff --git a/docs/reference/migration/migrate_7_0/discovery.asciidoc b/docs/reference/migration/migrate_7_0/discovery.asciidoc index d568e7fe32c25..193f6bdd86a6f 100644 --- a/docs/reference/migration/migrate_7_0/discovery.asciidoc +++ b/docs/reference/migration/migrate_7_0/discovery.asciidoc @@ -13,8 +13,8 @@ settings summary>> for an example, and the <> describes this setting in more detail. -The `discovery.zen.minimum_master_nodes` setting is required during a rolling -upgrade from 6.x, but can be removed in all other circumstances. +The `discovery.zen.minimum_master_nodes` setting is permitted, but ignored, on +7.x nodes. [float] ==== Removing master-eligible nodes sometimes requires voting exclusions diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java index cfa5a3f6d79c8..88afa57e83e23 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java @@ -33,9 +33,7 @@ import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.settings.Settings.Builder; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.http.HttpServerTransport; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalTestCluster; @@ -58,11 +56,7 @@ public class Zen2RestApiIT extends ESNetty4IntegTestCase { @Override protected Settings nodeSettings(int nodeOrdinal) { - final Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal)) - .put(TestZenDiscovery.USE_ZEN2.getKey(), true) - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE); - - return builder.build(); + return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put(TestZenDiscovery.USE_ZEN2.getKey(), true).build(); } @Override diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsTests.java index e91dff713b4f0..f11bd539fba7d 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsTests.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; +import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.discovery.DiscoveryModule; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; @@ -41,13 +42,14 @@ public void testMinimumMasterNodesStart() { .build(); internalCluster().startNode(nodeSettings); - // We try to update minimum_master_nodes now - ClusterUpdateSettingsResponse response = client().admin().cluster().prepareUpdateSettings() - .setPersistentSettings(Settings.builder().put("discovery.zen.minimum_master_nodes", 1)) - .setTransientSettings(Settings.builder().put("discovery.zen.minimum_master_nodes", 1)) + // We try to update a setting now + final String expectedValue = UUIDs.randomBase64UUID(random()); + final String settingName = "cluster.routing.allocation.exclude.any_attribute"; + final ClusterUpdateSettingsResponse response = client().admin().cluster().prepareUpdateSettings() + .setPersistentSettings(Settings.builder().put(settingName, expectedValue)) .get(); - Integer min = response.getPersistentSettings().getAsInt("discovery.zen.minimum_master_nodes", null); - assertThat(min, is(1)); + final String value = response.getPersistentSettings().get(settingName); + assertThat(value, is(expectedValue)); } } diff --git a/qa/logging-config/src/test/java/org/elasticsearch/qa/custom_logging/CustomLoggingConfigIT.java b/qa/logging-config/src/test/java/org/elasticsearch/qa/custom_logging/CustomLoggingConfigIT.java index 407d23de99769..2e6b32d79fc41 100644 --- a/qa/logging-config/src/test/java/org/elasticsearch/qa/custom_logging/CustomLoggingConfigIT.java +++ b/qa/logging-config/src/test/java/org/elasticsearch/qa/custom_logging/CustomLoggingConfigIT.java @@ -40,7 +40,7 @@ * The intention is to confirm that users can still run their Elasticsearch instances with previous configurations. */ public class CustomLoggingConfigIT extends ESRestTestCase { - private static final String NODE_STARTED = ".*node-0.*cluster.uuid.*node.id.*started.*"; + private static final String NODE_STARTED = ".*node-0.*cluster.uuid.*node.id.*recovered.*cluster_state.*"; public void testSuccessfulStartupWithCustomConfig() throws Exception { assertBusy(() -> { diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index 6894463dd6fa6..160edea6a7898 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -77,7 +77,6 @@ for (Version version : bwcVersions.wireCompatible) { dependsOn lastRunner, "${baseName}#oldClusterTestCluster#node${stopNode}.stop" clusterName = 'rolling-upgrade' otherUnicastHostAddresses = { getOtherUnicastHostAddresses() } - minimumMasterNodes = { 2 } autoSetInitialMasterNodes = false /* Override the data directory so the new node always gets the node we * just stopped's data directory. */ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml index d801f3aeac89f..825bac9f91649 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml @@ -4,22 +4,22 @@ cluster.put_settings: body: transient: - discovery.zen.minimum_master_nodes: 1 + cluster.routing.allocation.enable: "none" flat_settings: true - - match: {transient: {discovery.zen.minimum_master_nodes: "1"}} + - match: {transient: {cluster.routing.allocation.enable: "none"}} - do: cluster.get_settings: flat_settings: true - - match: {transient: {discovery.zen.minimum_master_nodes: "1"}} + - match: {transient: {cluster.routing.allocation.enable: "none"}} - do: cluster.put_settings: body: transient: - discovery.zen.minimum_master_nodes: null + cluster.routing.allocation.enable: null flat_settings: true - match: {transient: {}} @@ -35,22 +35,22 @@ cluster.put_settings: body: persistent: - cluster.routing.allocation.disk.threshold_enabled: false + cluster.routing.allocation.enable: "none" flat_settings: true - - match: {persistent: {cluster.routing.allocation.disk.threshold_enabled: "false"}} + - match: {persistent: {cluster.routing.allocation.enable: "none"}} - do: cluster.get_settings: flat_settings: true - - match: {persistent: {cluster.routing.allocation.disk.threshold_enabled: "false"}} + - match: {persistent: {cluster.routing.allocation.enable: "none"}} - do: cluster.put_settings: body: persistent: - cluster.routing.allocation.disk.threshold_enabled: null + cluster.routing.allocation.enable: null flat_settings: true - match: {persistent: {}} diff --git a/server/src/main/java/org/elasticsearch/discovery/zen/ElectMasterService.java b/server/src/main/java/org/elasticsearch/discovery/zen/ElectMasterService.java index 87ad0a396ca76..7e0f0cfca2a99 100644 --- a/server/src/main/java/org/elasticsearch/discovery/zen/ElectMasterService.java +++ b/server/src/main/java/org/elasticsearch/discovery/zen/ElectMasterService.java @@ -42,7 +42,7 @@ public class ElectMasterService { private static final Logger logger = LogManager.getLogger(ElectMasterService.class); public static final Setting DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING = - Setting.intSetting("discovery.zen.minimum_master_nodes", -1, Property.Dynamic, Property.NodeScope); + Setting.intSetting("discovery.zen.minimum_master_nodes", -1, Property.Dynamic, Property.NodeScope, Property.Deprecated); private volatile int minimumMasterNodes; diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsIT.java b/server/src/test/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsIT.java index cd90cda2ba286..33c0d22473c65 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsIT.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsIT.java @@ -23,7 +23,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.discovery.MasterNotDiscoveredException; -import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; @@ -51,8 +50,7 @@ protected List addExtraClusterBootstrapSettings(List allNode public void testIndexExistsWithBlocksInPlace() throws IOException { Settings settings = Settings.builder() - .put(GatewayService.RECOVER_AFTER_NODES_SETTING.getKey(), 99) - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE).build(); + .put(GatewayService.RECOVER_AFTER_NODES_SETTING.getKey(), 99).build(); String node = internalCluster().startNode(settings); assertThrows(client(node).admin().indices().prepareExists("test").setMasterNodeTimeout(TimeValue.timeValueSeconds(0)), diff --git a/server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java b/server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java index c16d9b11e0d91..26b8ae88d266d 100644 --- a/server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java @@ -26,14 +26,13 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.coordination.ClusterBootstrapService; +import org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.discovery.DiscoverySettings; -import org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException; -import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.discovery.zen.ZenDiscovery; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.node.Node; @@ -102,7 +101,6 @@ public void testTwoNodesNoMasterBlock() throws Exception { bootstrapNodeId = 2; Settings settings = Settings.builder() - .put("discovery.zen.minimum_master_nodes", 2) .put(ZenDiscovery.PING_TIMEOUT_SETTING.getKey(), "200ms") .put("discovery.initial_state_timeout", "500ms") .build(); @@ -237,7 +235,6 @@ public void testThreeNodesNoMasterBlock() throws Exception { bootstrapNodeId = 3; Settings settings = Settings.builder() - .put("discovery.zen.minimum_master_nodes", 3) .put(ZenDiscovery.PING_TIMEOUT_SETTING.getKey(), "1s") .put("discovery.initial_state_timeout", "500ms") .build(); @@ -316,11 +313,9 @@ public void testCannotCommitStateThreeNodes() throws Exception { Settings settings = Settings.builder() .put(ZenDiscovery.PING_TIMEOUT_SETTING.getKey(), "200ms") .put("discovery.initial_state_timeout", "500ms") - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 2) .put(DiscoverySettings.COMMIT_TIMEOUT_SETTING.getKey(), "100ms") // speed things up .build(); - internalCluster().startNodes(3, settings); ensureGreen(); // ensure cluster state is recovered before we disrupt things diff --git a/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java b/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java index aaf01b5e6e079..8758e169b5124 100644 --- a/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java @@ -38,7 +38,6 @@ import java.util.Collections; import java.util.List; -import static org.elasticsearch.discovery.zen.ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; @@ -47,12 +46,6 @@ @TestLogging("_root:DEBUG,org.elasticsearch.action.admin.cluster.state:TRACE") public class SpecificMasterNodesIT extends ESIntegTestCase { - @Override - protected Settings nodeSettings(int nodeOrdinal) { - return Settings.builder().put(super.nodeSettings(nodeOrdinal)) - .put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 1).build(); - } - @Override protected List addExtraClusterBootstrapSettings(List allNodesSettings) { // if it's the first master in the cluster bootstrap the cluster with this node name diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterIT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterIT.java index 73add5ba83520..334d392b1793d 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterIT.java @@ -28,7 +28,6 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.discovery.DiscoverySettings; -import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.NodeMetaData; @@ -155,7 +154,6 @@ public void testNoNodeMetaData() throws IOException { public void testNotBootstrappedCluster() throws Exception { internalCluster().startNode( Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) .put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s") // to ensure quick node startup .build()); assertBusy(() -> { @@ -172,9 +170,7 @@ public void testNotBootstrappedCluster() throws Exception { public void testNoManifestFile() throws IOException { bootstrapNodeId = 1; - internalCluster().startNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); + internalCluster().startNode(); ensureStableCluster(1); NodeEnvironment nodeEnvironment = internalCluster().getMasterNodeInstance(NodeEnvironment.class); internalCluster().stopRandomDataNode(); @@ -186,9 +182,7 @@ public void testNoManifestFile() throws IOException { public void testNoMetaData() throws IOException { bootstrapNodeId = 1; - internalCluster().startNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); + internalCluster().startNode(); ensureStableCluster(1); NodeEnvironment nodeEnvironment = internalCluster().getMasterNodeInstance(NodeEnvironment.class); internalCluster().stopRandomDataNode(); @@ -201,9 +195,7 @@ public void testNoMetaData() throws IOException { public void testAbortedByUser() throws IOException { bootstrapNodeId = 1; - internalCluster().startNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); + internalCluster().startNode(); ensureStableCluster(1); internalCluster().stopRandomDataNode(); @@ -213,13 +205,9 @@ public void testAbortedByUser() throws IOException { public void test3MasterNodes2Failed() throws Exception { bootstrapNodeId = 3; - List masterNodes = internalCluster().startMasterOnlyNodes(3, Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); + List masterNodes = internalCluster().startMasterOnlyNodes(3, Settings.EMPTY); - String dataNode = internalCluster().startDataOnlyNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); + String dataNode = internalCluster().startDataOnlyNode(); createIndex("test"); Client dataNodeClient = internalCluster().client(dataNode); @@ -246,9 +234,7 @@ public void test3MasterNodes2Failed() throws Exception { String.format(Locale.ROOT, UnsafeBootstrapMasterCommand.CLUSTER_STATE_TERM_VERSION_MSG_FORMAT, metaData.coordinationMetaData().term(), metaData.version()))); - internalCluster().startMasterOnlyNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); + internalCluster().startMasterOnlyNode(); assertBusy(() -> { ClusterState state = dataNodeClient.admin().cluster().prepareState().setLocal(true) diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/Zen1IT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/Zen1IT.java index eb4b2d75c73f9..e8cd691129745 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/Zen1IT.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/Zen1IT.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.gateway.MetaStateService; import org.elasticsearch.test.ESIntegTestCase; @@ -50,6 +51,7 @@ import static org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING; import static org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider.CLUSTER_ROUTING_EXCLUDE_GROUP_SETTING; import static org.elasticsearch.node.Node.NODE_NAME_SETTING; +import static org.elasticsearch.test.InternalTestCluster.REMOVED_MINIMUM_MASTER_NODES; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -194,7 +196,8 @@ public Settings onNodeStopped(String nodeName) { } ClusterHealthResponse clusterHealthResponse = clusterHealthRequestBuilder.get(); assertFalse(nodeName, clusterHealthResponse.isTimedOut()); - return Coordinator.addZen1Attribute(false, Settings.builder().put(ZEN2_SETTINGS)).build(); + return Coordinator.addZen1Attribute(false, Settings.builder().put(ZEN2_SETTINGS) + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), REMOVED_MINIMUM_MASTER_NODES)).build(); } }); @@ -289,6 +292,7 @@ public Settings onNodeStopped(String nodeName) throws Exception { return Coordinator.addZen1Attribute(false, Settings.builder()) .put(ZEN2_SETTINGS) .putList(INITIAL_MASTER_NODES_SETTING.getKey(), nodeNames) + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), REMOVED_MINIMUM_MASTER_NODES) .build(); } }); diff --git a/server/src/test/java/org/elasticsearch/discovery/MasterDisruptionIT.java b/server/src/test/java/org/elasticsearch/discovery/MasterDisruptionIT.java index 652ce1fca1e14..718904eecb5bb 100644 --- a/server/src/test/java/org/elasticsearch/discovery/MasterDisruptionIT.java +++ b/server/src/test/java/org/elasticsearch/discovery/MasterDisruptionIT.java @@ -35,7 +35,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.discovery.zen.ZenDiscovery; import org.elasticsearch.monitor.jvm.HotThreads; import org.elasticsearch.test.ESIntegTestCase; @@ -125,11 +124,6 @@ public void testNodesFDAfterMasterReelection() throws Exception { ensureStableCluster(3); - logger.info("--> reducing min master nodes to 2"); - assertAcked(client().admin().cluster().prepareUpdateSettings() - .setTransientSettings(Settings.builder().put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 2)) - .get()); - String master = internalCluster().getMasterName(); String nonMaster = null; for (String node : internalCluster().getNodeNames()) { @@ -154,7 +148,7 @@ public void testNodesFDAfterMasterReelection() throws Exception { */ @TestLogging("_root:DEBUG,org.elasticsearch.cluster.service:TRACE,org.elasticsearch.test.disruption:TRACE") public void testStaleMasterNotHijackingMajority() throws Exception { - // 3 node cluster with unicast discovery and minimum_master_nodes set to the default of 2: + // 3 node cluster with unicast discovery: final List nodes = startCluster(3); // Save the current master node as old master node, because that node will get frozen diff --git a/server/src/test/java/org/elasticsearch/discovery/zen/MinimumMasterNodesInClusterStateIT.java b/server/src/test/java/org/elasticsearch/discovery/zen/MinimumMasterNodesInClusterStateIT.java deleted file mode 100644 index f60a313a5d41e..0000000000000 --- a/server/src/test/java/org/elasticsearch/discovery/zen/MinimumMasterNodesInClusterStateIT.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.discovery.zen; - -import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.test.ESIntegTestCase; - -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.elasticsearch.discovery.zen.ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING; -import static org.elasticsearch.test.InternalTestCluster.nameFilter; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.isIn; - -@ESIntegTestCase.ClusterScope(numDataNodes = 0, numClientNodes = 0) -public class MinimumMasterNodesInClusterStateIT extends ESIntegTestCase { - - public void testMasterPublishes() throws Exception { - final String firstNode = internalCluster().startNode(); - - { - final ClusterState localState - = client(firstNode).admin().cluster().state(new ClusterStateRequest().local(true)).get().getState(); - assertThat(localState.getMinimumMasterNodesOnPublishingMaster(), equalTo(1)); - assertFalse(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(localState.metaData().settings())); - } - - final List secondThirdNodes = internalCluster().startNodes(2); - assertThat(internalCluster().getMasterName(), equalTo(firstNode)); - - final List allNodes = Stream.concat(Stream.of(firstNode), secondThirdNodes.stream()).collect(Collectors.toList()); - for (final String node : allNodes) { - final ClusterState localState = client(node).admin().cluster().state(new ClusterStateRequest().local(true)).get().getState(); - assertThat(localState.getMinimumMasterNodesOnPublishingMaster(), equalTo(1)); - assertThat(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(localState.metaData().settings()), equalTo(2)); - } - - internalCluster().stopRandomNode(nameFilter(firstNode)); - assertThat(internalCluster().getMasterName(), isIn(secondThirdNodes)); - - for (final String node : secondThirdNodes) { - final ClusterState localState = client(node).admin().cluster().state(new ClusterStateRequest().local(true)).get().getState(); - assertThat(localState.getMinimumMasterNodesOnPublishingMaster(), equalTo(2)); - assertThat(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(localState.metaData().settings()), equalTo(2)); - } - } -} diff --git a/server/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java b/server/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java index c9a2f7dc58388..084ba62c4792d 100644 --- a/server/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java +++ b/server/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java @@ -268,6 +268,9 @@ public void testNodesUpdatedAfterClusterStatePublished() throws Exception { IOUtils.close(toClose); terminate(threadPool); } + + assertWarnings("[discovery.zen.minimum_master_nodes] setting was deprecated in Elasticsearch and will be removed in a future " + + "release! See the breaking changes documentation for the next major version."); } public void testPendingCSQueueIsClearedWhenClusterStatePublished() throws Exception { @@ -318,6 +321,9 @@ public void testPendingCSQueueIsClearedWhenClusterStatePublished() throws Except IOUtils.close(toClose); terminate(threadPool); } + + assertWarnings("[discovery.zen.minimum_master_nodes] setting was deprecated in Elasticsearch and will be removed in a future " + + "release! See the breaking changes documentation for the next major version."); } private class AwaitingPublishListener implements ActionListener { diff --git a/server/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java b/server/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java index e97f69b6d4965..e6fc2ed975fbb 100644 --- a/server/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java @@ -25,7 +25,6 @@ import org.elasticsearch.cluster.coordination.ClusterBootstrapService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.node.Node; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; @@ -59,7 +58,7 @@ protected List addExtraClusterBootstrapSettings(List allNode return super.addExtraClusterBootstrapSettings(allNodesSettings); } - public Set waitForNoBlocksOnNode(TimeValue timeout, Client nodeClient) throws InterruptedException { + public Set waitForNoBlocksOnNode(TimeValue timeout, Client nodeClient) { long start = System.currentTimeMillis(); Set blocks; do { @@ -70,22 +69,20 @@ public Set waitForNoBlocksOnNode(TimeValue timeout, Client nodeCli return blocks; } - public Client startNode(Settings.Builder settings, int minMasterNodes) { - String name = internalCluster().startNode( - Settings.builder().put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minMasterNodes) - .put(settings.build())); + public Client startNode(Settings.Builder settings) { + String name = internalCluster().startNode(Settings.builder().put(settings.build())); return internalCluster().client(name); } public void testRecoverAfterNodes() throws Exception { logger.info("--> start node (1)"); - Client clientNode1 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3), 1); + Client clientNode1 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3)); assertThat(clientNode1.admin().cluster().prepareState().setLocal(true).execute().actionGet() .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE), hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK)); logger.info("--> start node (2)"); - Client clientNode2 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3), 1); + Client clientNode2 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3)); Thread.sleep(BLOCK_WAIT_TIMEOUT.millis()); assertThat(clientNode1.admin().cluster().prepareState().setLocal(true).execute().actionGet() .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE), @@ -95,7 +92,7 @@ public void testRecoverAfterNodes() throws Exception { hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK)); logger.info("--> start node (3)"); - Client clientNode3 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3), 1); + Client clientNode3 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3)); assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, clientNode1).isEmpty(), equalTo(true)); assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, clientNode2).isEmpty(), equalTo(true)); @@ -106,7 +103,7 @@ public void testRecoverAfterMasterNodes() throws Exception { logger.info("--> start master_node (1)"); Client master1 = startNode(Settings.builder() .put("gateway.recover_after_master_nodes", 2).put(Node.NODE_DATA_SETTING.getKey(), false) - .put(Node.NODE_MASTER_SETTING.getKey(), true), 1); + .put(Node.NODE_MASTER_SETTING.getKey(), true)); assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet() .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE), hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK)); @@ -114,7 +111,7 @@ public void testRecoverAfterMasterNodes() throws Exception { logger.info("--> start data_node (1)"); Client data1 = startNode(Settings.builder() .put("gateway.recover_after_master_nodes", 2) - .put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false), 1); + .put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false)); assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet() .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE), hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK)); @@ -125,7 +122,7 @@ public void testRecoverAfterMasterNodes() throws Exception { logger.info("--> start data_node (2)"); Client data2 = startNode(Settings.builder() .put("gateway.recover_after_master_nodes", 2).put(Node.NODE_DATA_SETTING.getKey(), true) - .put(Node.NODE_MASTER_SETTING.getKey(), false), 1); + .put(Node.NODE_MASTER_SETTING.getKey(), false)); assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet() .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE), hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK)); @@ -140,7 +137,7 @@ public void testRecoverAfterMasterNodes() throws Exception { Client master2 = startNode(Settings.builder() .put("gateway.recover_after_master_nodes", 2) .put(Node.NODE_DATA_SETTING.getKey(), false) - .put(Node.NODE_MASTER_SETTING.getKey(), true), 1); + .put(Node.NODE_MASTER_SETTING.getKey(), true)); assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, master1).isEmpty(), equalTo(true)); assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, master2).isEmpty(), equalTo(true)); assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, data1).isEmpty(), equalTo(true)); @@ -152,7 +149,7 @@ public void testRecoverAfterDataNodes() throws Exception { Client master1 = startNode(Settings.builder() .put("gateway.recover_after_data_nodes", 2) .put(Node.NODE_DATA_SETTING.getKey(), false) - .put(Node.NODE_MASTER_SETTING.getKey(), true), 1); + .put(Node.NODE_MASTER_SETTING.getKey(), true)); assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet() .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE), hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK)); @@ -161,7 +158,7 @@ public void testRecoverAfterDataNodes() throws Exception { Client data1 = startNode(Settings.builder() .put("gateway.recover_after_data_nodes", 2) .put(Node.NODE_DATA_SETTING.getKey(), true) - .put(Node.NODE_MASTER_SETTING.getKey(), false), 1); + .put(Node.NODE_MASTER_SETTING.getKey(), false)); assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet() .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE), hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK)); @@ -173,7 +170,7 @@ public void testRecoverAfterDataNodes() throws Exception { Client master2 = startNode(Settings.builder() .put("gateway.recover_after_data_nodes", 2) .put(Node.NODE_DATA_SETTING.getKey(), false) - .put(Node.NODE_MASTER_SETTING.getKey(), true), 1); + .put(Node.NODE_MASTER_SETTING.getKey(), true)); assertThat(master2.admin().cluster().prepareState().setLocal(true).execute().actionGet() .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE), hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK)); @@ -188,7 +185,7 @@ public void testRecoverAfterDataNodes() throws Exception { Client data2 = startNode(Settings.builder() .put("gateway.recover_after_data_nodes", 2) .put(Node.NODE_DATA_SETTING.getKey(), true) - .put(Node.NODE_MASTER_SETTING.getKey(), false), 1); + .put(Node.NODE_MASTER_SETTING.getKey(), false)); assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, master1).isEmpty(), equalTo(true)); assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, master2).isEmpty(), equalTo(true)); assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, data1).isEmpty(), equalTo(true)); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 6e3089a6d63e1..45f8682dc5e61 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -1087,17 +1087,6 @@ public long waitForDocs(final long numDocs, int maxWaitTime, TimeUnit maxWaitTim return lastKnownCount.get(); } - - /** - * Sets the cluster's minimum master node and make sure the response is acknowledge. - * Note: this doesn't guarantee that the new setting has taken effect, just that it has been received by all nodes. - */ - public void setMinimumMasterNodes(int n) { - assertTrue(client().admin().cluster().prepareUpdateSettings().setTransientSettings( - Settings.builder().put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), n)) - .get().isAcknowledged()); - } - /** * Prints the current cluster state as debug logging. */ diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index 55e356d093e6e..9313d9389d49c 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -33,8 +33,8 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsAction; import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest; -import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsRequest; import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsAction; +import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsRequest; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag; @@ -157,11 +157,15 @@ import static org.elasticsearch.test.ESTestCase.awaitBusy; import static org.elasticsearch.test.ESTestCase.getTestTransportType; import static org.elasticsearch.test.ESTestCase.randomFrom; +import static org.elasticsearch.test.discovery.TestZenDiscovery.USE_ZEN2; +import static org.elasticsearch.test.discovery.TestZenDiscovery.usingZen1; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -615,19 +619,28 @@ private Settings getNodeSettings(final int nodeId, final long seed, final Settin .put("node.name", name) .put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), seed); - final boolean usingSingleNodeDiscovery = DiscoveryModule.DISCOVERY_TYPE_SETTING.get(updatedSettings.build()).equals("single-node"); - if (!usingSingleNodeDiscovery && autoManageMinMasterNodes) { - assert updatedSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) == null : - "min master nodes may not be set when auto managed"; - assert updatedSettings.get(INITIAL_STATE_TIMEOUT_SETTING.getKey()) == null : - "automatically managing min master nodes require nodes to complete a join cycle" + - " when starting"; - updatedSettings - // don't wait too long not to slow down tests - .put(ZenDiscovery.MASTER_ELECTION_WAIT_FOR_JOINS_TIMEOUT_SETTING.getKey(), "5s") - .put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), defaultMinMasterNodes); - } else if (!usingSingleNodeDiscovery && updatedSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) == null) { - throw new IllegalArgumentException(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() + " must be configured"); + final String discoveryType = DiscoveryModule.DISCOVERY_TYPE_SETTING.get(updatedSettings.build()); + final boolean usingSingleNodeDiscovery = discoveryType.equals("single-node"); + final boolean usingZen1 = TestZenDiscovery.usingZen1(updatedSettings.build()); + if (usingSingleNodeDiscovery == false) { + if (autoManageMinMasterNodes) { + assertThat("min master nodes may not be set when auto managed", + updatedSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()), nullValue()); + assertThat("automatically managing min master nodes require nodes to complete a join cycle when starting", + updatedSettings.get(INITIAL_STATE_TIMEOUT_SETTING.getKey()), nullValue()); + + if (usingZen1) { + updatedSettings + // don't wait too long not to slow down tests + .put(ZenDiscovery.MASTER_ELECTION_WAIT_FOR_JOINS_TIMEOUT_SETTING.getKey(), "5s") + .put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), defaultMinMasterNodes); + } + } else { + if (usingZen1) { + assertThat(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() + " must be configured", + updatedSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()), not(nullValue())); + } + } } return updatedSettings.build(); @@ -822,6 +835,8 @@ public synchronized void close() { } } + public static final int REMOVED_MINIMUM_MASTER_NODES = Integer.MAX_VALUE; + private final class NodeAndClient implements Closeable { private MockNode node; private final Settings originalNodeSettings; @@ -933,8 +948,10 @@ Settings closeForRestart(RestartCallback callback, int minMasterNodes) throws Ex Settings.Builder newSettings = Settings.builder(); newSettings.put(callbackSettings); if (minMasterNodes >= 0) { - assert DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(newSettings.build()) == false : "min master nodes is auto managed"; - newSettings.put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minMasterNodes); + if (usingZen1(newSettings.build())) { + assertFalse("min master nodes is auto managed", DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(newSettings.build())); + newSettings.put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minMasterNodes); + } if (INITIAL_MASTER_NODES_SETTING.exists(callbackSettings) == false) { newSettings.putList(INITIAL_MASTER_NODES_SETTING.getKey()); } @@ -966,10 +983,23 @@ private void recreateNode(final Settings newSettings, final Runnable onTransport .put(newSettings) .put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), newIdSeed) .build(); - final boolean usingSingleNodeDiscovery = DiscoveryModule.DISCOVERY_TYPE_SETTING.get(finalSettings).equals("single-node"); - if (usingSingleNodeDiscovery == false && DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(finalSettings) == false) { - throw new IllegalStateException(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() + - " is not configured after restart of [" + name + "]"); + if (usingZen1(finalSettings)) { + if (DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(finalSettings) == false) { + throw new IllegalStateException(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() + + " is not configured after restart of [" + name + "]"); + } + } else { + if (DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(finalSettings)) { + // simulating an upgrade from Zen1 to Zen2, but there's no way to remove a setting when restarting a node, so + // you have to set it to REMOVED_MINIMUM_MASTER_NODES (== Integer.MAX_VALUE) to indicate its removal: + assertTrue(USE_ZEN2.exists(finalSettings)); + assertTrue(USE_ZEN2.get(finalSettings)); + assertThat(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(finalSettings), equalTo(REMOVED_MINIMUM_MASTER_NODES)); + + final Builder builder = Settings.builder().put(finalSettings); + builder.remove(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()); + finalSettings = builder.build(); + } } Collection> plugins = node.getClasspathPlugins(); node = new MockNode(finalSettings, plugins); diff --git a/test/framework/src/main/java/org/elasticsearch/test/discovery/TestZenDiscovery.java b/test/framework/src/main/java/org/elasticsearch/test/discovery/TestZenDiscovery.java index 53be34c0b40c8..56e6c24571715 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/discovery/TestZenDiscovery.java +++ b/test/framework/src/main/java/org/elasticsearch/test/discovery/TestZenDiscovery.java @@ -60,6 +60,7 @@ public class TestZenDiscovery extends ZenDiscovery { public static final Setting USE_ZEN2 = Setting.boolSetting("discovery.zen.use_zen2", true, Setting.Property.NodeScope); + private static final String TEST_ZEN_DISCOVERY_TYPE = "test-zen"; /** A plugin which installs mock discovery and configures it to be used. */ public static class TestPlugin extends Plugin implements DiscoveryPlugin { @@ -97,7 +98,7 @@ public List> getSettings() { @Override public Settings additionalSettings() { return Settings.builder() - .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "test-zen") + .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), TEST_ZEN_DISCOVERY_TYPE) .putList(DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.getKey()) .build(); } @@ -124,4 +125,9 @@ protected ZenPing newZenPing(Settings settings, ThreadPool threadPool, Transport public ZenPing getZenPing() { return zenPing; } + + public static boolean usingZen1(Settings settings) { + return DiscoveryModule.ZEN_DISCOVERY_TYPE.equals(DiscoveryModule.DISCOVERY_TYPE_SETTING.get(settings)) + || USE_ZEN2.get(settings) == false; + } } diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java index 38d76423eea9c..ca2fe8c753e44 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java @@ -63,7 +63,6 @@ import static org.elasticsearch.cluster.node.DiscoveryNode.Role.INGEST; import static org.elasticsearch.cluster.node.DiscoveryNode.Role.MASTER; import static org.elasticsearch.discovery.DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING; -import static org.elasticsearch.discovery.zen.ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING; import static org.elasticsearch.node.Node.NODE_MASTER_SETTING; import static org.elasticsearch.node.Node.NODE_NAME_SETTING; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileExists; @@ -101,7 +100,6 @@ public void testInitializiationIsConsistent() { nodePrefix, Collections.emptyList(), Function.identity()); // TODO: this is not ideal - we should have a way to make sure ports are initialized in the same way assertClusters(cluster0, cluster1, false); - } /** @@ -140,36 +138,6 @@ public static void assertSettings(Settings left, Settings right, boolean checkCl } } - private void assertMMNinNodeSetting(InternalTestCluster cluster, int masterNodes) { - for (final String node : cluster.getNodeNames()) { - assertMMNinNodeSetting(node, cluster, masterNodes); - } - } - - private void assertMMNinNodeSetting(String node, InternalTestCluster cluster, int masterNodes) { - final int minMasterNodes = masterNodes / 2 + 1; - Settings nodeSettings = cluster.client(node).admin().cluster().prepareNodesInfo(node).get().getNodes().get(0).getSettings(); - assertEquals("node setting of node [" + node + "] has the wrong min_master_node setting: [" - + nodeSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) + "]", - DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(nodeSettings).intValue(), minMasterNodes); - } - - private void assertMMNinClusterSetting(InternalTestCluster cluster, int masterNodes) { - for (final String node : cluster.getNodeNames()) { - assertMMNinClusterSetting(node, cluster, masterNodes); - } - } - - private void assertMMNinClusterSetting(String node, InternalTestCluster cluster, int masterNodes) { - final int minMasterNodes = masterNodes / 2 + 1; - Settings stateSettings = cluster.client(node).admin().cluster().prepareState().setLocal(true) - .get().getState().getMetaData().settings(); - - assertEquals("dynamic setting for node [" + node + "] has the wrong min_master_node setting : [" - + stateSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) + "]", - DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(stateSettings).intValue(), minMasterNodes); - } - public void testBeforeTest() throws Exception { final boolean autoManageMinMasterNodes = randomBoolean(); long clusterSeed = randomLong(); @@ -204,7 +172,6 @@ public Settings nodeSettings(int nodeOrdinal) { if (autoManageMinMasterNodes == false) { assert minNumDataNodes == maxNumDataNodes; assert masterNodes == false; - settings.put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minNumDataNodes / 2 + 1); } return settings.build(); } @@ -257,8 +224,6 @@ public Settings transportClientSettings() { Client other = iterator1.next(); assertSettings(client.settings(), other.settings(), false); } - assertMMNinNodeSetting(cluster0, cluster0.numMasterNodes()); - assertMMNinNodeSetting(cluster1, cluster0.numMasterNodes()); cluster0.afterTest(); cluster1.afterTest(); } finally { @@ -320,7 +285,6 @@ public Settings transportClientSettings() { try { cluster.beforeTest(random(), 0.0); final int originalMasterCount = cluster.numMasterNodes(); - assertMMNinNodeSetting(cluster, originalMasterCount); final Map shardNodePaths = new HashMap<>(); for (String name: cluster.getNodeNames()) { shardNodePaths.put(name, getNodePaths(cluster, name)); @@ -335,10 +299,6 @@ public Settings transportClientSettings() { expectedMasterCount--; } cluster.stopRandomNode(InternalTestCluster.nameFilter(poorNode)); - if (expectedMasterCount != originalMasterCount) { - // check for updated - assertMMNinClusterSetting(cluster, expectedMasterCount); - } assertFileExists(testMarker); // stopping a node half way shouldn't clean data final String stableNode = randomFrom(cluster.getNodeNames()); @@ -351,14 +311,8 @@ public Settings transportClientSettings() { expectedMasterCount++; assertThat(getNodePaths(cluster, newNode1)[0], equalTo(dataPath)); assertFileExists(testMarker); // starting a node should re-use data folders and not clean it - if (expectedMasterCount > 1) { // this is the first master, it's in cluster state settings won't be updated - assertMMNinClusterSetting(cluster, expectedMasterCount); - } - assertMMNinNodeSetting(newNode1, cluster, expectedMasterCount); - final String newNode2 = cluster.startNode(); expectedMasterCount++; - assertMMNinClusterSetting(cluster, expectedMasterCount); final Path newDataPath = getNodePaths(cluster, newNode2)[0]; final Path newTestMarker = newDataPath.resolve("newTestMarker"); assertThat(newDataPath, not(dataPath)); @@ -377,8 +331,6 @@ public Settings transportClientSettings() { assertThat("data paths for " + name + " changed", getNodePaths(cluster, name), equalTo(shardNodePaths.get(name))); } - assertMMNinNodeSetting(cluster, originalMasterCount); - } finally { cluster.close(); } @@ -393,7 +345,6 @@ private Path[] getNodePaths(InternalTestCluster cluster, String name) { } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37462") public void testDifferentRolesMaintainPathOnRestart() throws Exception { final Path baseDir = createTempDir(); final int numNodes = 5; @@ -448,11 +399,6 @@ public Settings transportClientSettings() { roles.add(role); } - final long masterCount = roles.stream().filter(role -> role == MASTER).count(); - final Settings minMasterNodes = Settings.builder() - .put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), masterCount / 2 + 1) - .build(); - try { Map> pathsPerRole = new HashMap<>(); for (int i = 0; i < numNodes; i++) { @@ -460,13 +406,13 @@ public Settings transportClientSettings() { final String node; switch (role) { case MASTER: - node = cluster.startMasterOnlyNode(minMasterNodes); + node = cluster.startMasterOnlyNode(); break; case DATA: - node = cluster.startDataOnlyNode(minMasterNodes); + node = cluster.startDataOnlyNode(); break; case INGEST: - node = cluster.startCoordinatingOnlyNode(minMasterNodes); + node = cluster.startCoordinatingOnlyNode(Settings.EMPTY); break; default: throw new IllegalStateException("get your story straight"); @@ -476,6 +422,7 @@ public Settings transportClientSettings() { assertTrue(rolePaths.add(path.toString())); } } + cluster.validateClusterFormed(); cluster.fullRestart(); Map> result = new HashMap<>(); @@ -533,34 +480,18 @@ public Settings transportClientSettings() { plugins, Function.identity()); try { cluster.beforeTest(random(), 0.0); - assertMMNinNodeSetting(cluster, 2); switch (randomInt(2)) { case 0: cluster.stopRandomDataNode(); - assertMMNinClusterSetting(cluster, 1); cluster.startNode(); - assertMMNinClusterSetting(cluster, 2); - assertMMNinNodeSetting(cluster, 2); break; case 1: - cluster.rollingRestart(new InternalTestCluster.RestartCallback() { - @Override - public Settings onNodeStopped(String nodeName) throws Exception { - for (String name : cluster.getNodeNames()) { - if (name.equals(nodeName) == false) { - assertMMNinClusterSetting(name, cluster, 1); - } - } - return super.onNodeStopped(nodeName); - } - }); - assertMMNinClusterSetting(cluster, 2); + cluster.rollingRestart(InternalTestCluster.EMPTY_CALLBACK); break; case 2: cluster.fullRestart(); break; } - assertMMNinNodeSetting(cluster, 2); } finally { cluster.close(); } @@ -585,6 +516,5 @@ public Settings additionalSettings() { } return Settings.builder().put("node.attr.dummy", true).build(); } - } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java index e79ec2e2fc059..f146f12245e1f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.discovery.DiscoveryModule; +import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.node.MockNode; import org.elasticsearch.node.Node; import org.elasticsearch.plugins.Plugin; @@ -290,23 +291,25 @@ public void testNodeJoinWithoutSecurityExplicitlyEnabled() throws Exception { Path home = createTempDir(); Path conf = home.resolve("config"); Files.createDirectories(conf); - Settings nodeSettings = Settings.builder() + Settings.Builder nodeSettings = Settings.builder() .put(nodeSettings(maxNumberOfNodes() - 1).filter(s -> "xpack.security.enabled".equals(s) == false)) .put("node.name", "my-test-node") .put("network.host", "localhost") .put("cluster.name", internalCluster().getClusterName()) - .put("discovery.zen.minimum_master_nodes", - internalCluster().getInstance(Settings.class).get("discovery.zen.minimum_master_nodes")) .put("path.home", home) .put(TestZenDiscovery.USE_MOCK_PINGS.getKey(), false) .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "test-zen") .put(TestZenDiscovery.USE_ZEN2.getKey(), getUseZen2()) .putList(DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING.getKey()) - .putList(DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.getKey(), unicastHostsList) - .build(); + .putList(DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.getKey(), unicastHostsList); + if (getUseZen2() == false) { + nodeSettings.put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), + ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(internalCluster().getInstance(Settings.class))); + } + Collection> mockPlugins = Arrays.asList(LocalStateSecurity.class, TestZenDiscovery.TestPlugin.class, MockHttpTransport.TestPlugin.class); - try (Node node = new MockNode(nodeSettings, mockPlugins)) { + try (Node node = new MockNode(nodeSettings.build(), mockPlugins)) { node.start(); ensureStableCluster(cluster().size() + 1); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterIntegrationTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterIntegrationTests.java index 2383f3b3ac739..83640d9e931b5 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterIntegrationTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterIntegrationTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.node.MockNode; import org.elasticsearch.node.Node; import org.elasticsearch.node.NodeValidationException; @@ -101,8 +102,6 @@ public void testThatConnectionToServerTypeConnectionWorks() throws IOException, .put("network.host", "localhost") .put("cluster.name", internalCluster().getClusterName()) .put("discovery.zen.ping.unicast.hosts", unicastHost) - .put("discovery.zen.minimum_master_nodes", - internalCluster().getInstance(Settings.class).get("discovery.zen.minimum_master_nodes")) .put("xpack.security.enabled", true) .put("xpack.security.audit.enabled", false) .put("xpack.security.transport.ssl.enabled", true) @@ -111,6 +110,10 @@ public void testThatConnectionToServerTypeConnectionWorks() throws IOException, .put(Node.NODE_MASTER_SETTING.getKey(), false) .put(TestZenDiscovery.USE_ZEN2.getKey(), getUseZen2()) .put(TestZenDiscovery.USE_MOCK_PINGS.getKey(), false); + if (getUseZen2() == false) { + nodeSettings.put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), + ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(internalCluster().getInstance(Settings.class))); + } Collection> mockPlugins = Arrays.asList( LocalStateSecurity.class, TestZenDiscovery.TestPlugin.class, MockHttpTransport.TestPlugin.class); addSSLSettingsForPEMFiles( @@ -144,8 +147,6 @@ public void testThatConnectionToClientTypeConnectionIsRejected() throws IOExcept .put(SecurityField.USER_SETTING.getKey(), "test_user:" + SecuritySettingsSourceField.TEST_PASSWORD) .put("cluster.name", internalCluster().getClusterName()) .put("discovery.zen.ping.unicast.hosts", unicastHost) - .put("discovery.zen.minimum_master_nodes", - internalCluster().getInstance(Settings.class).get("discovery.zen.minimum_master_nodes")) .put("xpack.security.enabled", true) .put("xpack.security.audit.enabled", false) .put("xpack.security.transport.ssl.enabled", true) @@ -155,6 +156,10 @@ public void testThatConnectionToClientTypeConnectionIsRejected() throws IOExcept .put(Node.NODE_MASTER_SETTING.getKey(), false) .put(TestZenDiscovery.USE_ZEN2.getKey(), getUseZen2()) .put(TestZenDiscovery.USE_MOCK_PINGS.getKey(), false); + if (getUseZen2() == false) { + nodeSettings.put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), + ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(internalCluster().getInstance(Settings.class))); + } Collection> mockPlugins = Arrays.asList( LocalStateSecurity.class, TestZenDiscovery.TestPlugin.class, MockHttpTransport.TestPlugin.class); addSSLSettingsForPEMFiles( From be788160efd23fed3eba5ef1d431cb1cb3e88cf6 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Wed, 30 Jan 2019 20:12:20 +0000 Subject: [PATCH 068/100] [ML] Datafeed deprecation checks (#38026) Deprecation checks for the ML datafeed query and aggregations. --- .../deprecation/DeprecationInfoAction.java | 48 +++++++++++++++---- .../core/ml/datafeed/DatafeedConfig.java | 2 +- .../DeprecationInfoActionResponseTests.java | 22 +++++++-- .../xpack/deprecation/DeprecationChecks.java | 7 +++ .../deprecation/MlDeprecationChecks.java | 44 +++++++++++++++++ .../TransportDeprecationInfoAction.java | 42 +++++++++++++--- .../deprecation/MlDeprecationChecksTests.java | 48 +++++++++++++++++++ .../test/deprecation/10_basic.yml | 46 ++++++++++++++++++ 8 files changed, 239 insertions(+), 20 deletions(-) create mode 100644 x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecks.java create mode 100644 x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecksTests.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoAction.java index 09c6a0d57524e..1241b136c7a6e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoAction.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.core.deprecation; +import org.elasticsearch.Version; import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; @@ -23,9 +24,12 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -67,16 +71,19 @@ public static class Response extends ActionResponse implements ToXContentObject private List clusterSettingsIssues; private List nodeSettingsIssues; private Map> indexSettingsIssues; + private List mlSettingsIssues; public Response() { } public Response(List clusterSettingsIssues, List nodeSettingsIssues, - Map> indexSettingsIssues) { + Map> indexSettingsIssues, + List mlSettingsIssues) { this.clusterSettingsIssues = clusterSettingsIssues; this.nodeSettingsIssues = nodeSettingsIssues; this.indexSettingsIssues = indexSettingsIssues; + this.mlSettingsIssues = mlSettingsIssues; } public List getClusterSettingsIssues() { @@ -91,12 +98,21 @@ public Map> getIndexSettingsIssues() { return indexSettingsIssues; } + public List getMlSettingsIssues() { + return mlSettingsIssues; + } + @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); clusterSettingsIssues = in.readList(DeprecationIssue::new); nodeSettingsIssues = in.readList(DeprecationIssue::new); indexSettingsIssues = in.readMapOfLists(StreamInput::readString, DeprecationIssue::new); + if (in.getVersion().onOrAfter(Version.V_6_7_0)) { + mlSettingsIssues = in.readList(DeprecationIssue::new); + } else { + mlSettingsIssues = Collections.emptyList(); + } } @Override @@ -105,6 +121,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeList(clusterSettingsIssues); out.writeList(nodeSettingsIssues); out.writeMapOfLists(indexSettingsIssues, StreamOutput::writeString, (o, v) -> v.writeTo(o)); + if (out.getVersion().onOrAfter(Version.V_6_7_0)) { + out.writeList(mlSettingsIssues); + } } @Override @@ -114,10 +133,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws .array("node_settings", nodeSettingsIssues.toArray()) .field("index_settings") .map(indexSettingsIssues) + .array("ml_settings", mlSettingsIssues.toArray()) .endObject(); } - @Override public boolean equals(Object o) { if (this == o) return true; @@ -125,12 +144,13 @@ public boolean equals(Object o) { Response response = (Response) o; return Objects.equals(clusterSettingsIssues, response.clusterSettingsIssues) && Objects.equals(nodeSettingsIssues, response.nodeSettingsIssues) && - Objects.equals(indexSettingsIssues, response.indexSettingsIssues); + Objects.equals(indexSettingsIssues, response.indexSettingsIssues) && + Objects.equals(mlSettingsIssues, response.mlSettingsIssues); } @Override public int hashCode() { - return Objects.hash(clusterSettingsIssues, nodeSettingsIssues, indexSettingsIssues); + return Objects.hash(clusterSettingsIssues, nodeSettingsIssues, indexSettingsIssues, mlSettingsIssues); } /** @@ -145,22 +165,30 @@ public int hashCode() { * @param indexNameExpressionResolver Used to resolve indices into their concrete names * @param indices The list of index expressions to evaluate using `indexNameExpressionResolver` * @param indicesOptions The options to use when resolving and filtering which indices to check + * @param datafeeds The ml datafeed configurations * @param clusterSettingsChecks The list of cluster-level checks * @param nodeSettingsChecks The list of node-level checks * @param indexSettingsChecks The list of index-level checks that will be run across all specified * concrete indices + * @param mlSettingsCheck The list of ml checks * @return The list of deprecation issues found in the cluster */ public static DeprecationInfoAction.Response from(List nodesInfo, List nodesStats, ClusterState state, - IndexNameExpressionResolver indexNameExpressionResolver, - String[] indices, IndicesOptions indicesOptions, - List>clusterSettingsChecks, - List, List, DeprecationIssue>> nodeSettingsChecks, - List> indexSettingsChecks) { + IndexNameExpressionResolver indexNameExpressionResolver, + String[] indices, IndicesOptions indicesOptions, + List datafeeds, + List>clusterSettingsChecks, + List, List, DeprecationIssue>> nodeSettingsChecks, + List> indexSettingsChecks, + List> mlSettingsCheck) { List clusterSettingsIssues = filterChecks(clusterSettingsChecks, (c) -> c.apply(state)); List nodeSettingsIssues = filterChecks(nodeSettingsChecks, (c) -> c.apply(nodesInfo, nodesStats)); + List mlSettingsIssues = new ArrayList<>(); + for (DatafeedConfig config : datafeeds) { + mlSettingsIssues.addAll(filterChecks(mlSettingsCheck, (c) -> c.apply(config))); + } String[] concreteIndexNames = indexNameExpressionResolver.concreteIndexNames(state, indicesOptions, indices); @@ -174,7 +202,7 @@ public static DeprecationInfoAction.Response from(List nodesInfo, List } } - return new DeprecationInfoAction.Response(clusterSettingsIssues, nodeSettingsIssues, indexSettingsIssues); + return new DeprecationInfoAction.Response(clusterSettingsIssues, nodeSettingsIssues, indexSettingsIssues, mlSettingsIssues); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java index 938452d27cc58..ed858b58dd484 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java @@ -665,7 +665,7 @@ public void setParsedQuery(QueryBuilder query) { } } - void setQuery(Map query) { + public void setQuery(Map query) { this.query = ExceptionsHelper.requireNonNull(query, QUERY.getPreferredName()); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoActionResponseTests.java index 5267e5dc2ff42..b878f1c5d404d 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoActionResponseTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoActionResponseTests.java @@ -22,6 +22,8 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.test.AbstractStreamableTestCase; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfigTests; import java.io.IOException; import java.util.Arrays; @@ -45,13 +47,15 @@ protected DeprecationInfoAction.Response createTestInstance() { .limit(randomIntBetween(0, 10)).collect(Collectors.toList()); List nodeIssues = Stream.generate(DeprecationIssueTests::createTestInstance) .limit(randomIntBetween(0, 10)).collect(Collectors.toList()); + List mlIssues = Stream.generate(DeprecationIssueTests::createTestInstance) + .limit(randomIntBetween(0, 10)).collect(Collectors.toList()); Map> indexIssues = new HashMap<>(); for (int i = 0; i < randomIntBetween(0, 10); i++) { List perIndexIssues = Stream.generate(DeprecationIssueTests::createTestInstance) .limit(randomIntBetween(0, 10)).collect(Collectors.toList()); indexIssues.put(randomAlphaOfLength(10), perIndexIssues); } - return new DeprecationInfoAction.Response(clusterIssues, nodeIssues, indexIssues); + return new DeprecationInfoAction.Response(clusterIssues, nodeIssues, indexIssues, mlIssues); } @Override @@ -80,12 +84,14 @@ public void testFrom() throws IOException { List nodeStats = Collections.singletonList(new NodeStats(discoveryNode, 0L, null, null, null, null, null, null, null, null, null, null, null, null, null)); + List datafeeds = Collections.singletonList(DatafeedConfigTests.createRandomizedDatafeedConfig("foo")); IndexNameExpressionResolver resolver = new IndexNameExpressionResolver(); IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, false, true, true); boolean clusterIssueFound = randomBoolean(); boolean nodeIssueFound = randomBoolean(); boolean indexIssueFound = randomBoolean(); + boolean mlIssueFound = randomBoolean(); DeprecationIssue foundIssue = DeprecationIssueTests.createTestInstance(); List> clusterSettingsChecks = Collections.unmodifiableList(Arrays.asList( @@ -100,10 +106,14 @@ public void testFrom() throws IOException { Collections.unmodifiableList(Arrays.asList( (idx) -> indexIssueFound ? foundIssue : null )); + List> mlSettingsChecks = + Collections.unmodifiableList(Arrays.asList( + (idx) -> mlIssueFound ? foundIssue : null + )); DeprecationInfoAction.Response response = DeprecationInfoAction.Response.from(nodeInfos, nodeStats, state, - resolver, Strings.EMPTY_ARRAY, indicesOptions, - clusterSettingsChecks, nodeSettingsChecks, indexSettingsChecks); + resolver, Strings.EMPTY_ARRAY, indicesOptions, datafeeds, + clusterSettingsChecks, nodeSettingsChecks, indexSettingsChecks, mlSettingsChecks); if (clusterIssueFound) { assertThat(response.getClusterSettingsIssues(), equalTo(Collections.singletonList(foundIssue))); @@ -123,5 +133,11 @@ public void testFrom() throws IOException { } else { assertTrue(response.getIndexSettingsIssues().isEmpty()); } + + if (mlIssueFound) { + assertThat(response.getMlSettingsIssues(), equalTo(Collections.singletonList(foundIssue))); + } else { + assertTrue(response.getMlSettingsIssues().isEmpty()); + } } } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java index 1363d3a09a03f..c6c3d5fd840c0 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java @@ -11,6 +11,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import java.util.Arrays; import java.util.Collections; @@ -41,6 +42,12 @@ private DeprecationChecks() { Collections.unmodifiableList(Arrays.asList( IndexDeprecationChecks::oldIndicesCheck)); + static List> ML_SETTINGS_CHECKS = + Collections.unmodifiableList(Arrays.asList( + MlDeprecationChecks::checkDataFeedAggregations, + MlDeprecationChecks::checkDataFeedQuery + )); + /** * helper utility function to reduce repeat of running a specific {@link List} of checks. * diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecks.java new file mode 100644 index 0000000000000..187a8669574cd --- /dev/null +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecks.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.deprecation; + +import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; + +import java.util.List; + +/** + * Check the {@link DatafeedConfig} query and aggregations for deprecated usages. + */ +final class MlDeprecationChecks { + + private MlDeprecationChecks() { + } + + static DeprecationIssue checkDataFeedQuery(DatafeedConfig datafeedConfig) { + List deprecations = datafeedConfig.getQueryDeprecations(); + if (deprecations.isEmpty()) { + return null; + } else { + return new DeprecationIssue(DeprecationIssue.Level.WARNING, + "Datafeed [" + datafeedConfig.getId() + "] uses deprecated query options", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html#breaking_70_search_changes", + deprecations.toString()); + } + } + + static DeprecationIssue checkDataFeedAggregations(DatafeedConfig datafeedConfig) { + List deprecations = datafeedConfig.getAggDeprecations(); + if (deprecations.isEmpty()) { + return null; + } else { + return new DeprecationIssue(DeprecationIssue.Level.WARNING, + "Datafeed [" + datafeedConfig.getId() + "] uses deprecated aggregation options", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + + "#breaking_70_aggregations_changes", deprecations.toString()); + } + } +} diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java index 29bd948ab6c33..6ae416248e9b7 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.license.XPackLicenseState; @@ -26,7 +27,13 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.XPackField; +import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction; +import org.elasticsearch.xpack.core.ml.action.GetDatafeedsAction; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; + +import java.util.Collections; +import java.util.List; public class TransportDeprecationInfoAction extends TransportMasterNodeReadAction { @@ -34,9 +41,10 @@ public class TransportDeprecationInfoAction extends TransportMasterNodeReadActio private final XPackLicenseState licenseState; private final NodeClient client; private final IndexNameExpressionResolver indexNameExpressionResolver; + private final Settings settings; @Inject - public TransportDeprecationInfoAction(TransportService transportService, ClusterService clusterService, + public TransportDeprecationInfoAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, XPackLicenseState licenseState, NodeClient client) { @@ -45,6 +53,7 @@ public TransportDeprecationInfoAction(TransportService transportService, Cluster this.licenseState = licenseState; this.client = client; this.indexNameExpressionResolver = indexNameExpressionResolver; + this.settings = settings; } @Override @@ -83,11 +92,20 @@ protected final void masterOperation(final DeprecationInfoAction.Request request if (nodesStatsResponse.hasFailures()) { throw nodesStatsResponse.failures().get(0); } - listener.onResponse(DeprecationInfoAction.Response.from(nodesInfoResponse.getNodes(), - nodesStatsResponse.getNodes(), state, indexNameExpressionResolver, - request.indices(), request.indicesOptions(), - DeprecationChecks.CLUSTER_SETTINGS_CHECKS, DeprecationChecks.NODE_SETTINGS_CHECKS, - DeprecationChecks.INDEX_SETTINGS_CHECKS)); + + getDatafeedConfigs(ActionListener.wrap( + datafeeds -> { + listener.onResponse( + DeprecationInfoAction.Response.from(nodesInfoResponse.getNodes(), + nodesStatsResponse.getNodes(), state, indexNameExpressionResolver, + request.indices(), request.indicesOptions(), datafeeds, + DeprecationChecks.CLUSTER_SETTINGS_CHECKS, + DeprecationChecks.NODE_SETTINGS_CHECKS, + DeprecationChecks.INDEX_SETTINGS_CHECKS, + DeprecationChecks.ML_SETTINGS_CHECKS)); + }, + listener::onFailure + )); }, listener::onFailure), client.admin().cluster()::nodesStats); }, listener::onFailure), client.admin().cluster()::nodesInfo); @@ -95,4 +113,16 @@ protected final void masterOperation(final DeprecationInfoAction.Request request listener.onFailure(LicenseUtils.newComplianceException(XPackField.DEPRECATION)); } } + + private void getDatafeedConfigs(ActionListener> listener) { + if (XPackSettings.MACHINE_LEARNING_ENABLED.get(settings) == false) { + listener.onResponse(Collections.emptyList()); + } else { + ClientHelper.executeAsyncWithOrigin(client, ClientHelper.DEPRECATION_ORIGIN, GetDatafeedsAction.INSTANCE, + new GetDatafeedsAction.Request(GetDatafeedsAction.ALL), ActionListener.wrap( + datafeedsResponse -> listener.onResponse(datafeedsResponse.getResponse().results()), + listener::onFailure + )); + } + } } diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecksTests.java new file mode 100644 index 0000000000000..6d93ed1873184 --- /dev/null +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecksTests.java @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.deprecation; + +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; + +import java.util.Collections; + +public class MlDeprecationChecksTests extends ESTestCase { + + @Override + protected boolean enableWarningsCheck() { + return false; + } + + public void testCheckDataFeedQuery() { + DatafeedConfig.Builder goodDatafeed = new DatafeedConfig.Builder("good-df", "job-id"); + goodDatafeed.setIndices(Collections.singletonList("some-index")); + goodDatafeed.setParsedQuery(new TermQueryBuilder("foo", "bar")); + assertNull(MlDeprecationChecks.checkDataFeedQuery(goodDatafeed.build())); + + DatafeedConfig.Builder deprecatedDatafeed = new DatafeedConfig.Builder("df-with-deprecated-query", "job-id"); + deprecatedDatafeed.setIndices(Collections.singletonList("some-index")); + // TODO: once some query syntax has been removed from 8.0 and deprecated in 7.x reinstate this test + // to check that particular query syntax causes a deprecation warning + /* + Map qs = new HashMap<>(); + qs.put("query", "foo"); + qs.put("use_dis_max", true); + Map query = Collections.singletonMap("query_string", qs); + deprecatedDatafeed.setQuery(query); + + DeprecationIssue issue = MlDeprecationChecks.checkDataFeedQuery(deprecatedDatafeed.build()); + assertNotNull(issue); + assertThat(issue.getDetails(), equalTo("[Deprecated field [use_dis_max] used, replaced by [Set [tie_breaker] to 1 instead]]")); + assertThat(issue.getLevel(), equalTo(DeprecationIssue.Level.WARNING)); + assertThat(issue.getMessage(), equalTo("Datafeed [df-with-deprecated-query] uses deprecated query options")); + assertThat(issue.getUrl(), equalTo("https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-8.0.html" + + "#breaking_80_search_changes")); + */ + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/deprecation/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/deprecation/10_basic.yml index dad0c3b08eb57..1cbb310bb4a08 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/deprecation/10_basic.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/deprecation/10_basic.yml @@ -12,4 +12,50 @@ setup: - length: { cluster_settings: 0 } - length: { node_settings: 0 } - length: { index_settings: 0 } + - length: { ml_settings: 0 } +--- +"Test ml": + - skip: + version: "7.0.0 - " + features: ["headers", "warnings"] + reason: this test needs adjusting to contain syntax deprecated in 7.x and removed in 8.0 + +# Index the config directly to prevent the deprecated +# use_dis_max field being rewritten by the parser. This +# simulates the config being created in an older version +# of elasticsearch + - do: + headers: + Content-Type: application/json + index: + index: .ml-config + type: doc + id: deprecation-datafeed-datafeed + body: > + { + "datafeed_id" : "deprecation-datafeed", + "config_type" : "datafeed", + "job_id" : "deprecation-job", + "indices" : ["index-foo"], + "query" : { + "query_string" : { + "query" : "foo", + "use_dis_max" : true + } + } + } + + - do: + indices.refresh: + index: [.ml-config] + +# TODO: change the query and expected warnings to one that makes sense for 7.x + - do: + warnings: + - Deprecated field [use_dis_max] used, replaced by [Set [tie_breaker] to 1 instead] + xpack.migration.deprecations: + index: "*" + - length: { ml_settings: 1 } + - match: { ml_settings.0.level : warning } + - match: { ml_settings.0.message : "Datafeed [deprecation-datafeed] uses deprecated query options" } From 945ad05d54c3dabc5ab0a39e6bac6dd287f2b00b Mon Sep 17 00:00:00 2001 From: Michael Basnight Date: Wed, 30 Jan 2019 14:31:16 -0600 Subject: [PATCH 069/100] Update verify repository to allow unknown fields (#37619) The subparser in verify repository allows for unknown fields. This commit sets the value to true for the parser and modifies the test such that it accurately tests it. Relates #36938 --- .../VerifyRepositoryResponseTests.java | 59 +++++++++++++++++++ .../verify/VerifyRepositoryResponse.java | 24 ++++---- 2 files changed, 71 insertions(+), 12 deletions(-) create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/VerifyRepositoryResponseTests.java diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/VerifyRepositoryResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/VerifyRepositoryResponseTests.java new file mode 100644 index 0000000000000..72193dc55e9e1 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/VerifyRepositoryResponseTests.java @@ -0,0 +1,59 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.watcher; + +import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyRepositoryResponse; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester; + +public class VerifyRepositoryResponseTests extends ESTestCase { + + public void testFromXContent() throws IOException { + xContentTester(this::createParser, + VerifyRepositoryResponseTests::createTestInstance, + VerifyRepositoryResponseTests::toXContent, + VerifyRepositoryResponse::fromXContent) + .supportsUnknownFields(true) + .shuffleFieldsExceptions(new String[] {"nodes"}) // do not mix up the order of nodes, it will cause the tests to fail + .randomFieldsExcludeFilter((f) -> f.equals("nodes")) // everything in nodes needs to be a particular parseable object + .assertToXContentEquivalence(false) + .test(); + } + + private static VerifyRepositoryResponse createTestInstance() { + List nodes = new ArrayList<>(); + for (int i = 0; i < randomIntBetween(0, 2); i++) { + nodes.add(new VerifyRepositoryResponse.NodeView(randomAlphaOfLength(5), randomAlphaOfLength(5))); + } + + return new VerifyRepositoryResponse(nodes); + } + + private static XContentBuilder toXContent(VerifyRepositoryResponse response, XContentBuilder builder) throws IOException { + return response.toXContent(builder, ToXContent.EMPTY_PARAMS); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java index 41835d3e11255..d72136852631f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java @@ -48,7 +48,7 @@ public class VerifyRepositoryResponse extends ActionResponse implements ToXConte public static class NodeView implements Writeable, ToXContentObject { private static final ObjectParser.NamedObjectParser PARSER; static { - ObjectParser internalParser = new ObjectParser<>(NODES); + ObjectParser internalParser = new ObjectParser<>(NODES, true, null); internalParser.declareString(NodeView::setName, new ParseField(NAME)); PARSER = (p, v, name) -> internalParser.parse(p, new NodeView(name), null); } @@ -110,7 +110,7 @@ public int hashCode() { private List nodes; private static final ObjectParser PARSER = - new ObjectParser<>(VerifyRepositoryResponse.class.getName(), VerifyRepositoryResponse::new); + new ObjectParser<>(VerifyRepositoryResponse.class.getName(), true, VerifyRepositoryResponse::new); static { PARSER.declareNamedObjects(VerifyRepositoryResponse::setNodes, NodeView.PARSER, new ParseField("nodes")); } @@ -122,6 +122,10 @@ public VerifyRepositoryResponse(DiscoveryNode[] nodes) { this.nodes = Arrays.stream(nodes).map(dn -> new NodeView(dn.getId(), dn.getName())).collect(Collectors.toList()); } + public VerifyRepositoryResponse(List nodes) { + this.nodes = nodes; + } + @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); @@ -168,19 +172,15 @@ public String toString() { } @Override - public boolean equals(Object obj) { - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - VerifyRepositoryResponse other = (VerifyRepositoryResponse) obj; - return nodes.equals(other.nodes); + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + VerifyRepositoryResponse that = (VerifyRepositoryResponse) o; + return Objects.equals(nodes, that.nodes); } @Override public int hashCode() { - return nodes.hashCode(); + return Objects.hash(nodes); } } From 14c571532a1557424e02d05382c758233134e781 Mon Sep 17 00:00:00 2001 From: Michael Basnight Date: Wed, 30 Jan 2019 14:31:55 -0600 Subject: [PATCH 070/100] Fix ILM Lifecycle Policy to allow unknown fields (#38041) A few of the ILM Lifecycle Policy and classes did not allow for unknown fields. This commit sets those parsers and fixes the tests for the parsers. Relates #36938 --- .../client/indexlifecycle/AllocateAction.java | 2 +- .../client/indexlifecycle/DeleteAction.java | 2 +- .../indexlifecycle/ForceMergeAction.java | 2 +- .../client/indexlifecycle/FreezeAction.java | 2 +- .../indexlifecycle/LifecyclePolicy.java | 2 +- .../LifecyclePolicyMetadata.java | 3 ++- .../client/indexlifecycle/Phase.java | 2 +- .../indexlifecycle/PhaseExecutionInfo.java | 2 +- .../client/indexlifecycle/ReadOnlyAction.java | 2 +- .../client/indexlifecycle/RolloverAction.java | 2 +- .../client/indexlifecycle/ShrinkAction.java | 2 +- .../client/indexlifecycle/UnfollowAction.java | 2 +- .../indexlifecycle/AllocateActionTests.java | 10 +++++++++- .../indexlifecycle/DeleteActionTests.java | 2 +- .../indexlifecycle/ForceMergeActionTests.java | 2 +- .../indexlifecycle/FreezeActionTests.java | 2 +- .../GetLifecyclePolicyResponseTests.java | 19 ++++++++++++++++++- .../LifecyclePolicyMetadataTests.java | 17 ++++++++++++++++- .../indexlifecycle/LifecyclePolicyTests.java | 9 ++++++++- .../PhaseExecutionInfoTests.java | 9 ++++++++- .../client/indexlifecycle/PhaseTests.java | 9 ++++++++- .../indexlifecycle/ReadOnlyActionTests.java | 2 +- .../indexlifecycle/RolloverActionTests.java | 2 +- .../indexlifecycle/ShrinkActionTests.java | 2 +- .../indexlifecycle/UnfollowActionTests.java | 2 +- 25 files changed, 87 insertions(+), 25 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/AllocateAction.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/AllocateAction.java index 702db15b965c7..a297e9b550e4e 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/AllocateAction.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/AllocateAction.java @@ -40,7 +40,7 @@ public class AllocateAction implements LifecycleAction, ToXContentObject { static final ParseField REQUIRE_FIELD = new ParseField("require"); @SuppressWarnings("unchecked") - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, true, a -> new AllocateAction((Integer) a[0], (Map) a[1], (Map) a[2], (Map) a[3])); static { diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/DeleteAction.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/DeleteAction.java index 299b0ac582771..9592b2edda0e4 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/DeleteAction.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/DeleteAction.java @@ -30,7 +30,7 @@ public class DeleteAction implements LifecycleAction, ToXContentObject { public static final String NAME = "delete"; - private static final ObjectParser PARSER = new ObjectParser<>(NAME, DeleteAction::new); + private static final ObjectParser PARSER = new ObjectParser<>(NAME, true, DeleteAction::new); public static DeleteAction parse(XContentParser parser) { return PARSER.apply(parser, null); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ForceMergeAction.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ForceMergeAction.java index eb564b7cd27b6..8b05b16eebafe 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ForceMergeAction.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ForceMergeAction.java @@ -33,7 +33,7 @@ public class ForceMergeAction implements LifecycleAction, ToXContentObject { private static final ParseField MAX_NUM_SEGMENTS_FIELD = new ParseField("max_num_segments"); private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, - false, a -> { + true, a -> { int maxNumSegments = (int) a[0]; return new ForceMergeAction(maxNumSegments); }); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/FreezeAction.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/FreezeAction.java index ecc054c132d67..3e5952539bb89 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/FreezeAction.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/FreezeAction.java @@ -29,7 +29,7 @@ public class FreezeAction implements LifecycleAction, ToXContentObject { public static final String NAME = "freeze"; - private static final ObjectParser PARSER = new ObjectParser<>(NAME, FreezeAction::new); + private static final ObjectParser PARSER = new ObjectParser<>(NAME, true, FreezeAction::new); public static FreezeAction parse(XContentParser parser) { return PARSER.apply(parser, null); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicy.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicy.java index 5e4ae1f36bcbc..9032afd7ba417 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicy.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicy.java @@ -44,7 +44,7 @@ public class LifecyclePolicy implements ToXContentObject { static final ParseField PHASES_FIELD = new ParseField("phases"); @SuppressWarnings("unchecked") - public static ConstructingObjectParser PARSER = new ConstructingObjectParser<>("lifecycle_policy", false, + public static ConstructingObjectParser PARSER = new ConstructingObjectParser<>("lifecycle_policy", true, (a, name) -> { List phases = (List) a[0]; Map phaseMap = phases.stream().collect(Collectors.toMap(Phase::getName, Function.identity())); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyMetadata.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyMetadata.java index 84de81437065d..b58594e5756c9 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyMetadata.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyMetadata.java @@ -38,7 +38,8 @@ public class LifecyclePolicyMetadata implements ToXContentObject { static final ParseField MODIFIED_DATE = new ParseField("modified_date"); @SuppressWarnings("unchecked") - public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("policy_metadata", + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "policy_metadata", true, a -> { LifecyclePolicy policy = (LifecyclePolicy) a[0]; return new LifecyclePolicyMetadata(policy, (long) a[1], ZonedDateTime.parse((String) a[2]).toInstant().toEpochMilli()); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/Phase.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/Phase.java index 0c19d39c85964..f6d3e80644764 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/Phase.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/Phase.java @@ -44,7 +44,7 @@ public class Phase implements ToXContentObject { static final ParseField ACTIONS_FIELD = new ParseField("actions"); @SuppressWarnings("unchecked") - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("phase", false, + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("phase", true, (a, name) -> new Phase(name, (TimeValue) a[0], ((List) a[1]).stream() .collect(Collectors.toMap(LifecycleAction::getName, Function.identity())))); static { diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/PhaseExecutionInfo.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/PhaseExecutionInfo.java index 802ca8834cdd3..681f79c67829c 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/PhaseExecutionInfo.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/PhaseExecutionInfo.java @@ -40,7 +40,7 @@ public class PhaseExecutionInfo implements ToXContentObject { private static final ParseField MODIFIED_DATE_IN_MILLIS_FIELD = new ParseField("modified_date_in_millis"); private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - "phase_execution_info", false, + "phase_execution_info", true, (a, name) -> new PhaseExecutionInfo((String) a[0], (Phase) a[1], (long) a[2], (long) a[3])); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), POLICY_NAME_FIELD); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ReadOnlyAction.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ReadOnlyAction.java index 7734e792bbc5b..cf364af6d9e5b 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ReadOnlyAction.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ReadOnlyAction.java @@ -29,7 +29,7 @@ public class ReadOnlyAction implements LifecycleAction, ToXContentObject { public static final String NAME = "readonly"; - private static final ObjectParser PARSER = new ObjectParser<>(NAME, false, ReadOnlyAction::new); + private static final ObjectParser PARSER = new ObjectParser<>(NAME, true, ReadOnlyAction::new); public static ReadOnlyAction parse(XContentParser parser) { return PARSER.apply(parser, null); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/RolloverAction.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/RolloverAction.java index 0cc9dcf234969..e84cc6921440a 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/RolloverAction.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/RolloverAction.java @@ -38,7 +38,7 @@ public class RolloverAction implements LifecycleAction, ToXContentObject { private static final ParseField MAX_DOCS_FIELD = new ParseField("max_docs"); private static final ParseField MAX_AGE_FIELD = new ParseField("max_age"); - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, true, a -> new RolloverAction((ByteSizeValue) a[0], (TimeValue) a[1], (Long) a[2])); static { PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ShrinkAction.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ShrinkAction.java index 345356380145e..920fe4a46257f 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ShrinkAction.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ShrinkAction.java @@ -33,7 +33,7 @@ public class ShrinkAction implements LifecycleAction, ToXContentObject { private static final ParseField NUMBER_OF_SHARDS_FIELD = new ParseField("number_of_shards"); private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>(NAME, a -> new ShrinkAction((Integer) a[0])); + new ConstructingObjectParser<>(NAME, true, a -> new ShrinkAction((Integer) a[0])); static { PARSER.declareInt(ConstructingObjectParser.constructorArg(), NUMBER_OF_SHARDS_FIELD); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/UnfollowAction.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/UnfollowAction.java index ba25cf937ec8f..11e9836efc02d 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/UnfollowAction.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/UnfollowAction.java @@ -31,7 +31,7 @@ public class UnfollowAction implements LifecycleAction, ToXContentObject { public static final String NAME = "unfollow"; - private static final ObjectParser PARSER = new ObjectParser<>(NAME, UnfollowAction::new); + private static final ObjectParser PARSER = new ObjectParser<>(NAME, true, UnfollowAction::new); public UnfollowAction() {} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/AllocateActionTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/AllocateActionTests.java index e44eb0da0e188..1cbfdf4f369cc 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/AllocateActionTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/AllocateActionTests.java @@ -24,6 +24,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.function.Predicate; public class AllocateActionTests extends AbstractXContentTestCase { @@ -65,7 +66,14 @@ protected AllocateAction doParseInstance(XContentParser parser) { @Override protected boolean supportsUnknownFields() { - return false; + return true; + } + + @Override + protected Predicate getRandomFieldsExcludeFilter() { + // this whole structure expects to be maps of strings, so more complex objects would just mess that up. + // setting it this way allows for new fields at the root + return (field) -> field.isEmpty() == false; } public void testAllMapsNullOrEmpty() { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/DeleteActionTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/DeleteActionTests.java index fb7deb97a2787..9f4dcf6bc641a 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/DeleteActionTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/DeleteActionTests.java @@ -35,6 +35,6 @@ protected DeleteAction doParseInstance(XContentParser parser) { @Override protected boolean supportsUnknownFields() { - return false; + return true; } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ForceMergeActionTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ForceMergeActionTests.java index 16fafcfa24015..7c3181a61dfd3 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ForceMergeActionTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ForceMergeActionTests.java @@ -39,7 +39,7 @@ protected ForceMergeAction doParseInstance(XContentParser parser) { @Override protected boolean supportsUnknownFields() { - return false; + return true; } @Override diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/FreezeActionTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/FreezeActionTests.java index 3fc40ee137b53..1a92db1cdf1f5 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/FreezeActionTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/FreezeActionTests.java @@ -35,6 +35,6 @@ protected FreezeAction doParseInstance(XContentParser parser) { @Override protected boolean supportsUnknownFields() { - return false; + return true; } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/GetLifecyclePolicyResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/GetLifecyclePolicyResponseTests.java index c16c270512ca6..ff1733498e37d 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/GetLifecyclePolicyResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/GetLifecyclePolicyResponseTests.java @@ -30,6 +30,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.function.Predicate; import static org.elasticsearch.client.indexlifecycle.LifecyclePolicyTests.createRandomPolicy; @@ -54,7 +55,23 @@ protected GetLifecyclePolicyResponse doParseInstance(XContentParser parser) thro @Override protected boolean supportsUnknownFields() { - return false; + return true; + } + + @Override + protected Predicate getRandomFieldsExcludeFilter() { + return (field) -> + // phases is a list of Phase parsable entries only + field.endsWith(".phases") + // these are all meant to be maps of strings, so complex objects will confuse the parser + || field.endsWith(".include") + || field.endsWith(".exclude") + || field.endsWith(".require") + // actions are meant to be a list of LifecycleAction parsable entries only + || field.endsWith(".actions") + // field.isEmpty() means do not insert an object at the root of the json. This parser expects + // every root level named object to be parsable as a specific type + || field.isEmpty(); } @Override diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyMetadataTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyMetadataTests.java index 6d8014c432c28..eba3113f1777d 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyMetadataTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyMetadataTests.java @@ -29,6 +29,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.function.Predicate; import static org.elasticsearch.client.indexlifecycle.LifecyclePolicyTests.createRandomPolicy; @@ -50,7 +51,21 @@ protected LifecyclePolicyMetadata doParseInstance(XContentParser parser) throws @Override protected boolean supportsUnknownFields() { - return false; + return true; + } + + @Override + protected Predicate getRandomFieldsExcludeFilter() { + return (field) -> + // phases is a list of Phase parsable entries only + field.endsWith(".phases") + // these are all meant to be maps of strings, so complex objects will confuse the parser + || field.endsWith(".include") + || field.endsWith(".exclude") + || field.endsWith(".require") + // actions are meant to be a list of LifecycleAction parsable entries only + || field.endsWith(".actions"); + } @Override diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyTests.java index 1690f66572142..6451b4bbbef59 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyTests.java @@ -34,6 +34,7 @@ import java.util.Map; import java.util.Set; import java.util.function.Function; +import java.util.function.Predicate; import java.util.stream.Collectors; import static org.hamcrest.Matchers.equalTo; @@ -55,7 +56,13 @@ protected LifecyclePolicy doParseInstance(XContentParser parser) { @Override protected boolean supportsUnknownFields() { - return false; + return true; + } + + @Override + protected Predicate getRandomFieldsExcludeFilter() { + // these items all have some specific parsing that does not allow them to have additional objects within them. + return (field) -> field.contains("allocate.") || field.equals("phases") || field.endsWith("actions"); } @Override diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/PhaseExecutionInfoTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/PhaseExecutionInfoTests.java index 0db9b56aea93c..fea740a442c95 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/PhaseExecutionInfoTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/PhaseExecutionInfoTests.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.function.Predicate; public class PhaseExecutionInfoTests extends AbstractXContentTestCase { @@ -53,9 +54,15 @@ protected PhaseExecutionInfo doParseInstance(XContentParser parser) throws IOExc return PhaseExecutionInfo.parse(parser, phaseName); } + @Override + protected Predicate getRandomFieldsExcludeFilter() { + // actions are plucked from the named registry, and it fails if the action is not in the named registry + return (field) -> field.equals("phase_definition.actions"); + } + @Override protected boolean supportsUnknownFields() { - return false; + return true; } @Override diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/PhaseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/PhaseTests.java index 3b4fc2fec6059..df4f11d18d028 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/PhaseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/PhaseTests.java @@ -30,6 +30,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Predicate; public class PhaseTests extends AbstractXContentTestCase { private String phaseName; @@ -61,6 +62,12 @@ protected Phase doParseInstance(XContentParser parser) { return Phase.parse(parser, phaseName); } + @Override + protected Predicate getRandomFieldsExcludeFilter() { + // actions are plucked from the named registry, and it fails if the action is not in the named registry + return (field) -> field.equals("actions"); + } + @Override protected NamedXContentRegistry xContentRegistry() { List entries = new ArrayList<>(ClusterModule.getNamedXWriteables()); @@ -70,7 +77,7 @@ protected NamedXContentRegistry xContentRegistry() { @Override protected boolean supportsUnknownFields() { - return false; + return true; } public void testDefaultAfter() { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ReadOnlyActionTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ReadOnlyActionTests.java index bf57478425cc9..dd6f62a447b39 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ReadOnlyActionTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ReadOnlyActionTests.java @@ -30,7 +30,7 @@ protected ReadOnlyAction doParseInstance(XContentParser parser) { @Override protected boolean supportsUnknownFields() { - return false; + return true; } @Override diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/RolloverActionTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/RolloverActionTests.java index bbbdba37e5640..833321d702c63 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/RolloverActionTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/RolloverActionTests.java @@ -33,7 +33,7 @@ protected RolloverAction doParseInstance(XContentParser parser) { @Override protected boolean supportsUnknownFields() { - return false; + return true; } @Override diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ShrinkActionTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ShrinkActionTests.java index adeec1ff825a9..d796221518fe1 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ShrinkActionTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ShrinkActionTests.java @@ -43,7 +43,7 @@ static ShrinkAction randomInstance() { @Override protected boolean supportsUnknownFields() { - return false; + return true; } public void testNonPositiveShardNumber() { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/UnfollowActionTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/UnfollowActionTests.java index 4dd73c5a08ec2..715a692213232 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/UnfollowActionTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/UnfollowActionTests.java @@ -38,6 +38,6 @@ protected UnfollowAction doParseInstance(XContentParser parser) throws IOExcepti @Override protected boolean supportsUnknownFields() { - return false; + return true; } } From daafcb66256aa209673b3c5b7bc57af8ed51a3b5 Mon Sep 17 00:00:00 2001 From: Michael Basnight Date: Wed, 30 Jan 2019 14:32:17 -0600 Subject: [PATCH 071/100] Fix ILM status to allow unknown fields (#38043) The ILM status parser did not allow for unknown fields. This commit fixes that and adds an xContentTester to the response test. Relates #36938 --- .../LifecycleManagementStatusResponse.java | 2 +- ...ifecycleManagementStatusResponseTests.java | 24 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecycleManagementStatusResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecycleManagementStatusResponse.java index c1586d7e1c738..d084113853f32 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecycleManagementStatusResponse.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecycleManagementStatusResponse.java @@ -34,7 +34,7 @@ public class LifecycleManagementStatusResponse { private static final String OPERATION_MODE = "operation_mode"; @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - OPERATION_MODE, a -> new LifecycleManagementStatusResponse((String) a[0])); + OPERATION_MODE, true, a -> new LifecycleManagementStatusResponse((String) a[0])); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField(OPERATION_MODE)); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecycleManagementStatusResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecycleManagementStatusResponseTests.java index 144039b8995c6..d027454453aca 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecycleManagementStatusResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecycleManagementStatusResponseTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; @@ -30,8 +31,31 @@ import java.util.EnumSet; import java.util.stream.Collectors; +import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester; + public class LifecycleManagementStatusResponseTests extends ESTestCase { + public void testFromXContent() throws IOException { + xContentTester(this::createParser, + LifecycleManagementStatusResponseTests::createTestInstance, + LifecycleManagementStatusResponseTests::toXContent, + LifecycleManagementStatusResponse::fromXContent) + .supportsUnknownFields(true) + .assertToXContentEquivalence(false) + .test(); + } + + private static XContentBuilder toXContent(LifecycleManagementStatusResponse response, XContentBuilder builder) throws IOException { + builder.startObject(); + builder.field("operation_mode", response.getOperationMode()); + builder.endObject(); + return builder; + } + + private static LifecycleManagementStatusResponse createTestInstance() { + return new LifecycleManagementStatusResponse(randomFrom(OperationMode.values()).name()); + } + public void testAllValidStatuses() { EnumSet.allOf(OperationMode.class) .forEach(e -> assertEquals(new LifecycleManagementStatusResponse(e.name()).getOperationMode(), e)); From aeab55e8d125efdfe66e4c9ad784c9d8c7fe6736 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Wed, 30 Jan 2019 14:13:23 -0700 Subject: [PATCH 072/100] Reduce flaxiness of ccr recovery timeouts test (#38035) This fixes #38027. Currently we assert that all shards have failed. However, it is possible that some shards do not have segement files created yet. The action that we block is fetching these segement files so it is possible that some shards successfully recover. This commit changes the assertion to ensure that at least some of the shards have failed. --- .../java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java index f22857939e0d1..fdf2de6d6775f 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java @@ -55,7 +55,9 @@ import static org.elasticsearch.snapshots.RestoreService.restoreInProgress; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.lessThan; // TODO: Fold this integration test into a more expansive integration test as more bootstrap from remote work // TODO: is completed. @@ -367,8 +369,9 @@ public void testIndividualActionsTimeout() throws Exception { // be marked as failed. Either one is a success for the purpose of this test. try { RestoreInfo restoreInfo = future.actionGet(); - assertEquals(0, restoreInfo.successfulShards()); - assertEquals(numberOfPrimaryShards, restoreInfo.failedShards()); + assertThat(restoreInfo.failedShards(), greaterThan(0)); + assertThat(restoreInfo.successfulShards(), lessThan(restoreInfo.totalShards())); + assertEquals(numberOfPrimaryShards, restoreInfo.totalShards()); } catch (Exception e) { assertThat(ExceptionsHelper.unwrapCause(e), instanceOf(ElasticsearchTimeoutException.class)); } From 54dbf9469cb31b9b832f3e02243329a8dd0235b6 Mon Sep 17 00:00:00 2001 From: Jay Modi Date: Wed, 30 Jan 2019 14:24:29 -0700 Subject: [PATCH 073/100] Update httpclient for JDK 11 TLS engine (#37994) The apache commons http client implementations recently released versions that solve TLS compatibility issues with the new TLS engine that supports TLSv1.3 with JDK 11. This change updates our code to use these versions since JDK 11 is a supported JDK and we should allow the use of TLSv1.3. --- buildSrc/version.properties | 11 +++---- .../rest/licenses/commons-codec-1.10.jar.sha1 | 1 - .../rest/licenses/commons-codec-1.11.jar.sha1 | 1 + .../licenses/httpasyncclient-4.1.2.jar.sha1 | 1 - .../licenses/httpasyncclient-4.1.4.jar.sha1 | 1 + .../rest/licenses/httpclient-4.5.2.jar.sha1 | 1 - .../rest/licenses/httpclient-4.5.7.jar.sha1 | 1 + client/rest/licenses/httpcore-4.4.11.jar.sha1 | 1 + client/rest/licenses/httpcore-4.4.5.jar.sha1 | 1 - .../licenses/httpcore-nio-4.4.11.jar.sha1 | 1 + .../rest/licenses/httpcore-nio-4.4.5.jar.sha1 | 1 - .../licenses/commons-codec-1.10.jar.sha1 | 1 - .../licenses/commons-codec-1.11.jar.sha1 | 1 + .../licenses/httpclient-4.5.2.jar.sha1 | 1 - .../licenses/httpclient-4.5.7.jar.sha1 | 1 + .../sniffer/licenses/httpcore-4.4.11.jar.sha1 | 1 + .../sniffer/licenses/httpcore-4.4.5.jar.sha1 | 1 - .../licenses/commons-codec-1.10.jar.sha1 | 1 - .../licenses/commons-codec-1.11.jar.sha1 | 1 + .../licenses/commons-codec-1.10.jar.sha1 | 1 - .../licenses/commons-codec-1.11.jar.sha1 | 1 + .../licenses/httpclient-4.5.2.jar.sha1 | 1 - .../licenses/httpclient-4.5.7.jar.sha1 | 1 + .../licenses/httpcore-4.4.11.jar.sha1 | 1 + .../licenses/httpcore-4.4.5.jar.sha1 | 1 - .../licenses/commons-codec-1.10.jar.sha1 | 1 - .../licenses/commons-codec-1.11.jar.sha1 | 1 + .../licenses/httpclient-4.5.2.jar.sha1 | 1 - .../licenses/httpclient-4.5.7.jar.sha1 | 1 + .../licenses/httpcore-4.4.11.jar.sha1 | 1 + .../licenses/httpcore-4.4.5.jar.sha1 | 1 - .../licenses/commons-codec-1.10.jar.sha1 | 1 - .../licenses/commons-codec-1.11.jar.sha1 | 1 + .../licenses/httpclient-4.5.2.jar.sha1 | 1 - .../licenses/httpclient-4.5.7.jar.sha1 | 1 + .../licenses/httpcore-4.4.11.jar.sha1 | 1 + .../licenses/httpcore-4.4.5.jar.sha1 | 1 - .../licenses/commons-codec-1.10.jar.sha1 | 1 - .../licenses/commons-codec-1.11.jar.sha1 | 1 + .../licenses/commons-codec-1.10.jar.sha1 | 1 - .../licenses/commons-codec-1.11.jar.sha1 | 1 + .../licenses/httpclient-4.5.2.jar.sha1 | 1 - .../licenses/httpclient-4.5.7.jar.sha1 | 1 + .../licenses/httpcore-4.4.11.jar.sha1 | 1 + .../licenses/httpcore-4.4.5.jar.sha1 | 1 - plugins/repository-hdfs/build.gradle | 2 +- .../licenses/commons-codec-1.10.jar.sha1 | 1 - .../licenses/commons-codec-1.11.jar.sha1 | 1 + .../licenses/commons-codec-1.10.jar.sha1 | 1 - .../licenses/commons-codec-1.11.jar.sha1 | 1 + .../licenses/httpclient-4.5.2.jar.sha1 | 1 - .../licenses/httpclient-4.5.7.jar.sha1 | 1 + .../licenses/httpcore-4.4.11.jar.sha1 | 1 + .../licenses/httpcore-4.4.5.jar.sha1 | 1 - .../core/licenses/commons-codec-1.10.jar.sha1 | 1 - .../core/licenses/commons-codec-1.11.jar.sha1 | 1 + .../licenses/httpasyncclient-4.1.2.jar.sha1 | 1 - .../licenses/httpasyncclient-4.1.4.jar.sha1 | 1 + .../core/licenses/httpclient-4.5.2.jar.sha1 | 1 - .../core/licenses/httpclient-4.5.7.jar.sha1 | 1 + .../core/licenses/httpcore-4.4.11.jar.sha1 | 1 + .../core/licenses/httpcore-4.4.5.jar.sha1 | 1 - .../licenses/httpcore-nio-4.4.11.jar.sha1 | 1 + .../core/licenses/httpcore-nio-4.4.5.jar.sha1 | 1 - .../licenses/httpclient-cache-4.5.2.jar.sha1 | 1 - .../licenses/httpclient-cache-4.5.7.jar.sha1 | 1 + .../xpack/watcher/common/http/HttpClient.java | 29 +++++++++++++++---- 67 files changed, 61 insertions(+), 45 deletions(-) delete mode 100644 client/rest/licenses/commons-codec-1.10.jar.sha1 create mode 100644 client/rest/licenses/commons-codec-1.11.jar.sha1 delete mode 100644 client/rest/licenses/httpasyncclient-4.1.2.jar.sha1 create mode 100644 client/rest/licenses/httpasyncclient-4.1.4.jar.sha1 delete mode 100644 client/rest/licenses/httpclient-4.5.2.jar.sha1 create mode 100644 client/rest/licenses/httpclient-4.5.7.jar.sha1 create mode 100644 client/rest/licenses/httpcore-4.4.11.jar.sha1 delete mode 100644 client/rest/licenses/httpcore-4.4.5.jar.sha1 create mode 100644 client/rest/licenses/httpcore-nio-4.4.11.jar.sha1 delete mode 100644 client/rest/licenses/httpcore-nio-4.4.5.jar.sha1 delete mode 100644 client/sniffer/licenses/commons-codec-1.10.jar.sha1 create mode 100644 client/sniffer/licenses/commons-codec-1.11.jar.sha1 delete mode 100644 client/sniffer/licenses/httpclient-4.5.2.jar.sha1 create mode 100644 client/sniffer/licenses/httpclient-4.5.7.jar.sha1 create mode 100644 client/sniffer/licenses/httpcore-4.4.11.jar.sha1 delete mode 100644 client/sniffer/licenses/httpcore-4.4.5.jar.sha1 delete mode 100644 plugins/analysis-phonetic/licenses/commons-codec-1.10.jar.sha1 create mode 100644 plugins/analysis-phonetic/licenses/commons-codec-1.11.jar.sha1 delete mode 100644 plugins/discovery-azure-classic/licenses/commons-codec-1.10.jar.sha1 create mode 100644 plugins/discovery-azure-classic/licenses/commons-codec-1.11.jar.sha1 delete mode 100644 plugins/discovery-azure-classic/licenses/httpclient-4.5.2.jar.sha1 create mode 100644 plugins/discovery-azure-classic/licenses/httpclient-4.5.7.jar.sha1 create mode 100644 plugins/discovery-azure-classic/licenses/httpcore-4.4.11.jar.sha1 delete mode 100644 plugins/discovery-azure-classic/licenses/httpcore-4.4.5.jar.sha1 delete mode 100644 plugins/discovery-ec2/licenses/commons-codec-1.10.jar.sha1 create mode 100644 plugins/discovery-ec2/licenses/commons-codec-1.11.jar.sha1 delete mode 100644 plugins/discovery-ec2/licenses/httpclient-4.5.2.jar.sha1 create mode 100644 plugins/discovery-ec2/licenses/httpclient-4.5.7.jar.sha1 create mode 100644 plugins/discovery-ec2/licenses/httpcore-4.4.11.jar.sha1 delete mode 100644 plugins/discovery-ec2/licenses/httpcore-4.4.5.jar.sha1 delete mode 100644 plugins/discovery-gce/licenses/commons-codec-1.10.jar.sha1 create mode 100644 plugins/discovery-gce/licenses/commons-codec-1.11.jar.sha1 delete mode 100644 plugins/discovery-gce/licenses/httpclient-4.5.2.jar.sha1 create mode 100644 plugins/discovery-gce/licenses/httpclient-4.5.7.jar.sha1 create mode 100644 plugins/discovery-gce/licenses/httpcore-4.4.11.jar.sha1 delete mode 100644 plugins/discovery-gce/licenses/httpcore-4.4.5.jar.sha1 delete mode 100644 plugins/ingest-attachment/licenses/commons-codec-1.10.jar.sha1 create mode 100644 plugins/ingest-attachment/licenses/commons-codec-1.11.jar.sha1 delete mode 100644 plugins/repository-gcs/licenses/commons-codec-1.10.jar.sha1 create mode 100644 plugins/repository-gcs/licenses/commons-codec-1.11.jar.sha1 delete mode 100644 plugins/repository-gcs/licenses/httpclient-4.5.2.jar.sha1 create mode 100644 plugins/repository-gcs/licenses/httpclient-4.5.7.jar.sha1 create mode 100644 plugins/repository-gcs/licenses/httpcore-4.4.11.jar.sha1 delete mode 100644 plugins/repository-gcs/licenses/httpcore-4.4.5.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/commons-codec-1.10.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/commons-codec-1.11.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/commons-codec-1.10.jar.sha1 create mode 100644 plugins/repository-s3/licenses/commons-codec-1.11.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/httpclient-4.5.2.jar.sha1 create mode 100644 plugins/repository-s3/licenses/httpclient-4.5.7.jar.sha1 create mode 100644 plugins/repository-s3/licenses/httpcore-4.4.11.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/httpcore-4.4.5.jar.sha1 delete mode 100644 x-pack/plugin/core/licenses/commons-codec-1.10.jar.sha1 create mode 100644 x-pack/plugin/core/licenses/commons-codec-1.11.jar.sha1 delete mode 100644 x-pack/plugin/core/licenses/httpasyncclient-4.1.2.jar.sha1 create mode 100644 x-pack/plugin/core/licenses/httpasyncclient-4.1.4.jar.sha1 delete mode 100644 x-pack/plugin/core/licenses/httpclient-4.5.2.jar.sha1 create mode 100644 x-pack/plugin/core/licenses/httpclient-4.5.7.jar.sha1 create mode 100644 x-pack/plugin/core/licenses/httpcore-4.4.11.jar.sha1 delete mode 100644 x-pack/plugin/core/licenses/httpcore-4.4.5.jar.sha1 create mode 100644 x-pack/plugin/core/licenses/httpcore-nio-4.4.11.jar.sha1 delete mode 100644 x-pack/plugin/core/licenses/httpcore-nio-4.4.5.jar.sha1 delete mode 100644 x-pack/plugin/security/licenses/httpclient-cache-4.5.2.jar.sha1 create mode 100644 x-pack/plugin/security/licenses/httpclient-cache-4.5.7.jar.sha1 diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 778f29686ad67..118ab2f905f74 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -21,16 +21,13 @@ joda = 2.10.1 # test dependencies randomizedrunner = 2.7.1 junit = 4.12 -httpclient = 4.5.2 -# When updating httpcore, please also update server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy -httpcore = 4.4.5 -# When updating httpasyncclient, please also update server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy -httpasyncclient = 4.1.2 +httpclient = 4.5.7 +httpcore = 4.4.11 +httpasyncclient = 4.1.4 commonslogging = 1.1.3 -commonscodec = 1.10 +commonscodec = 1.11 hamcrest = 1.3 securemock = 1.2 -# When updating mocksocket, please also update server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy mocksocket = 1.2 # benchmark dependencies diff --git a/client/rest/licenses/commons-codec-1.10.jar.sha1 b/client/rest/licenses/commons-codec-1.10.jar.sha1 deleted file mode 100644 index 3fe8682a1b0f9..0000000000000 --- a/client/rest/licenses/commons-codec-1.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4b95f4897fa13f2cd904aee711aeafc0c5295cd8 \ No newline at end of file diff --git a/client/rest/licenses/commons-codec-1.11.jar.sha1 b/client/rest/licenses/commons-codec-1.11.jar.sha1 new file mode 100644 index 0000000000000..b08f71a5babf0 --- /dev/null +++ b/client/rest/licenses/commons-codec-1.11.jar.sha1 @@ -0,0 +1 @@ +3acb4705652e16236558f0f4f2192cc33c3bd189 \ No newline at end of file diff --git a/client/rest/licenses/httpasyncclient-4.1.2.jar.sha1 b/client/rest/licenses/httpasyncclient-4.1.2.jar.sha1 deleted file mode 100644 index 065ed920a1773..0000000000000 --- a/client/rest/licenses/httpasyncclient-4.1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -95aa3e6fb520191a0970a73cf09f62948ee614be \ No newline at end of file diff --git a/client/rest/licenses/httpasyncclient-4.1.4.jar.sha1 b/client/rest/licenses/httpasyncclient-4.1.4.jar.sha1 new file mode 100644 index 0000000000000..8360ab45c7ab3 --- /dev/null +++ b/client/rest/licenses/httpasyncclient-4.1.4.jar.sha1 @@ -0,0 +1 @@ +f3a3240681faae3fa46b573a4c7e50cec9db0d86 \ No newline at end of file diff --git a/client/rest/licenses/httpclient-4.5.2.jar.sha1 b/client/rest/licenses/httpclient-4.5.2.jar.sha1 deleted file mode 100644 index 6937112a09fb6..0000000000000 --- a/client/rest/licenses/httpclient-4.5.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -733db77aa8d9b2d68015189df76ab06304406e50 \ No newline at end of file diff --git a/client/rest/licenses/httpclient-4.5.7.jar.sha1 b/client/rest/licenses/httpclient-4.5.7.jar.sha1 new file mode 100644 index 0000000000000..a8b7cc0d994d3 --- /dev/null +++ b/client/rest/licenses/httpclient-4.5.7.jar.sha1 @@ -0,0 +1 @@ +dda059f4908e1b548b7ba68d81a3b05897f27cb0 \ No newline at end of file diff --git a/client/rest/licenses/httpcore-4.4.11.jar.sha1 b/client/rest/licenses/httpcore-4.4.11.jar.sha1 new file mode 100644 index 0000000000000..6d64372bfccd8 --- /dev/null +++ b/client/rest/licenses/httpcore-4.4.11.jar.sha1 @@ -0,0 +1 @@ +de748cf874e4e193b42eceea9fe5574fabb9d4df \ No newline at end of file diff --git a/client/rest/licenses/httpcore-4.4.5.jar.sha1 b/client/rest/licenses/httpcore-4.4.5.jar.sha1 deleted file mode 100644 index 581726601745b..0000000000000 --- a/client/rest/licenses/httpcore-4.4.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e7501a1b34325abb00d17dde96150604a0658b54 \ No newline at end of file diff --git a/client/rest/licenses/httpcore-nio-4.4.11.jar.sha1 b/client/rest/licenses/httpcore-nio-4.4.11.jar.sha1 new file mode 100644 index 0000000000000..9e8777cb3da1c --- /dev/null +++ b/client/rest/licenses/httpcore-nio-4.4.11.jar.sha1 @@ -0,0 +1 @@ +7d0a97d01d39cff9aa3e6db81f21fddb2435f4e6 \ No newline at end of file diff --git a/client/rest/licenses/httpcore-nio-4.4.5.jar.sha1 b/client/rest/licenses/httpcore-nio-4.4.5.jar.sha1 deleted file mode 100644 index d6a80bf100de3..0000000000000 --- a/client/rest/licenses/httpcore-nio-4.4.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f4be009e7505f6ceddf21e7960c759f413f15056 \ No newline at end of file diff --git a/client/sniffer/licenses/commons-codec-1.10.jar.sha1 b/client/sniffer/licenses/commons-codec-1.10.jar.sha1 deleted file mode 100644 index 3fe8682a1b0f9..0000000000000 --- a/client/sniffer/licenses/commons-codec-1.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4b95f4897fa13f2cd904aee711aeafc0c5295cd8 \ No newline at end of file diff --git a/client/sniffer/licenses/commons-codec-1.11.jar.sha1 b/client/sniffer/licenses/commons-codec-1.11.jar.sha1 new file mode 100644 index 0000000000000..b08f71a5babf0 --- /dev/null +++ b/client/sniffer/licenses/commons-codec-1.11.jar.sha1 @@ -0,0 +1 @@ +3acb4705652e16236558f0f4f2192cc33c3bd189 \ No newline at end of file diff --git a/client/sniffer/licenses/httpclient-4.5.2.jar.sha1 b/client/sniffer/licenses/httpclient-4.5.2.jar.sha1 deleted file mode 100644 index 6937112a09fb6..0000000000000 --- a/client/sniffer/licenses/httpclient-4.5.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -733db77aa8d9b2d68015189df76ab06304406e50 \ No newline at end of file diff --git a/client/sniffer/licenses/httpclient-4.5.7.jar.sha1 b/client/sniffer/licenses/httpclient-4.5.7.jar.sha1 new file mode 100644 index 0000000000000..a8b7cc0d994d3 --- /dev/null +++ b/client/sniffer/licenses/httpclient-4.5.7.jar.sha1 @@ -0,0 +1 @@ +dda059f4908e1b548b7ba68d81a3b05897f27cb0 \ No newline at end of file diff --git a/client/sniffer/licenses/httpcore-4.4.11.jar.sha1 b/client/sniffer/licenses/httpcore-4.4.11.jar.sha1 new file mode 100644 index 0000000000000..6d64372bfccd8 --- /dev/null +++ b/client/sniffer/licenses/httpcore-4.4.11.jar.sha1 @@ -0,0 +1 @@ +de748cf874e4e193b42eceea9fe5574fabb9d4df \ No newline at end of file diff --git a/client/sniffer/licenses/httpcore-4.4.5.jar.sha1 b/client/sniffer/licenses/httpcore-4.4.5.jar.sha1 deleted file mode 100644 index 581726601745b..0000000000000 --- a/client/sniffer/licenses/httpcore-4.4.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e7501a1b34325abb00d17dde96150604a0658b54 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/commons-codec-1.10.jar.sha1 b/plugins/analysis-phonetic/licenses/commons-codec-1.10.jar.sha1 deleted file mode 100644 index ebd32cee72347..0000000000000 --- a/plugins/analysis-phonetic/licenses/commons-codec-1.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4b95f4897fa13f2cd904aee711aeafc0c5295cd8 diff --git a/plugins/analysis-phonetic/licenses/commons-codec-1.11.jar.sha1 b/plugins/analysis-phonetic/licenses/commons-codec-1.11.jar.sha1 new file mode 100644 index 0000000000000..b08f71a5babf0 --- /dev/null +++ b/plugins/analysis-phonetic/licenses/commons-codec-1.11.jar.sha1 @@ -0,0 +1 @@ +3acb4705652e16236558f0f4f2192cc33c3bd189 \ No newline at end of file diff --git a/plugins/discovery-azure-classic/licenses/commons-codec-1.10.jar.sha1 b/plugins/discovery-azure-classic/licenses/commons-codec-1.10.jar.sha1 deleted file mode 100644 index 3fe8682a1b0f9..0000000000000 --- a/plugins/discovery-azure-classic/licenses/commons-codec-1.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4b95f4897fa13f2cd904aee711aeafc0c5295cd8 \ No newline at end of file diff --git a/plugins/discovery-azure-classic/licenses/commons-codec-1.11.jar.sha1 b/plugins/discovery-azure-classic/licenses/commons-codec-1.11.jar.sha1 new file mode 100644 index 0000000000000..b08f71a5babf0 --- /dev/null +++ b/plugins/discovery-azure-classic/licenses/commons-codec-1.11.jar.sha1 @@ -0,0 +1 @@ +3acb4705652e16236558f0f4f2192cc33c3bd189 \ No newline at end of file diff --git a/plugins/discovery-azure-classic/licenses/httpclient-4.5.2.jar.sha1 b/plugins/discovery-azure-classic/licenses/httpclient-4.5.2.jar.sha1 deleted file mode 100644 index 6937112a09fb6..0000000000000 --- a/plugins/discovery-azure-classic/licenses/httpclient-4.5.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -733db77aa8d9b2d68015189df76ab06304406e50 \ No newline at end of file diff --git a/plugins/discovery-azure-classic/licenses/httpclient-4.5.7.jar.sha1 b/plugins/discovery-azure-classic/licenses/httpclient-4.5.7.jar.sha1 new file mode 100644 index 0000000000000..a8b7cc0d994d3 --- /dev/null +++ b/plugins/discovery-azure-classic/licenses/httpclient-4.5.7.jar.sha1 @@ -0,0 +1 @@ +dda059f4908e1b548b7ba68d81a3b05897f27cb0 \ No newline at end of file diff --git a/plugins/discovery-azure-classic/licenses/httpcore-4.4.11.jar.sha1 b/plugins/discovery-azure-classic/licenses/httpcore-4.4.11.jar.sha1 new file mode 100644 index 0000000000000..6d64372bfccd8 --- /dev/null +++ b/plugins/discovery-azure-classic/licenses/httpcore-4.4.11.jar.sha1 @@ -0,0 +1 @@ +de748cf874e4e193b42eceea9fe5574fabb9d4df \ No newline at end of file diff --git a/plugins/discovery-azure-classic/licenses/httpcore-4.4.5.jar.sha1 b/plugins/discovery-azure-classic/licenses/httpcore-4.4.5.jar.sha1 deleted file mode 100644 index 581726601745b..0000000000000 --- a/plugins/discovery-azure-classic/licenses/httpcore-4.4.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e7501a1b34325abb00d17dde96150604a0658b54 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/commons-codec-1.10.jar.sha1 b/plugins/discovery-ec2/licenses/commons-codec-1.10.jar.sha1 deleted file mode 100644 index 3fe8682a1b0f9..0000000000000 --- a/plugins/discovery-ec2/licenses/commons-codec-1.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4b95f4897fa13f2cd904aee711aeafc0c5295cd8 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/commons-codec-1.11.jar.sha1 b/plugins/discovery-ec2/licenses/commons-codec-1.11.jar.sha1 new file mode 100644 index 0000000000000..b08f71a5babf0 --- /dev/null +++ b/plugins/discovery-ec2/licenses/commons-codec-1.11.jar.sha1 @@ -0,0 +1 @@ +3acb4705652e16236558f0f4f2192cc33c3bd189 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/httpclient-4.5.2.jar.sha1 b/plugins/discovery-ec2/licenses/httpclient-4.5.2.jar.sha1 deleted file mode 100644 index 6937112a09fb6..0000000000000 --- a/plugins/discovery-ec2/licenses/httpclient-4.5.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -733db77aa8d9b2d68015189df76ab06304406e50 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/httpclient-4.5.7.jar.sha1 b/plugins/discovery-ec2/licenses/httpclient-4.5.7.jar.sha1 new file mode 100644 index 0000000000000..a8b7cc0d994d3 --- /dev/null +++ b/plugins/discovery-ec2/licenses/httpclient-4.5.7.jar.sha1 @@ -0,0 +1 @@ +dda059f4908e1b548b7ba68d81a3b05897f27cb0 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/httpcore-4.4.11.jar.sha1 b/plugins/discovery-ec2/licenses/httpcore-4.4.11.jar.sha1 new file mode 100644 index 0000000000000..6d64372bfccd8 --- /dev/null +++ b/plugins/discovery-ec2/licenses/httpcore-4.4.11.jar.sha1 @@ -0,0 +1 @@ +de748cf874e4e193b42eceea9fe5574fabb9d4df \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/httpcore-4.4.5.jar.sha1 b/plugins/discovery-ec2/licenses/httpcore-4.4.5.jar.sha1 deleted file mode 100644 index 581726601745b..0000000000000 --- a/plugins/discovery-ec2/licenses/httpcore-4.4.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e7501a1b34325abb00d17dde96150604a0658b54 \ No newline at end of file diff --git a/plugins/discovery-gce/licenses/commons-codec-1.10.jar.sha1 b/plugins/discovery-gce/licenses/commons-codec-1.10.jar.sha1 deleted file mode 100644 index 3fe8682a1b0f9..0000000000000 --- a/plugins/discovery-gce/licenses/commons-codec-1.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4b95f4897fa13f2cd904aee711aeafc0c5295cd8 \ No newline at end of file diff --git a/plugins/discovery-gce/licenses/commons-codec-1.11.jar.sha1 b/plugins/discovery-gce/licenses/commons-codec-1.11.jar.sha1 new file mode 100644 index 0000000000000..b08f71a5babf0 --- /dev/null +++ b/plugins/discovery-gce/licenses/commons-codec-1.11.jar.sha1 @@ -0,0 +1 @@ +3acb4705652e16236558f0f4f2192cc33c3bd189 \ No newline at end of file diff --git a/plugins/discovery-gce/licenses/httpclient-4.5.2.jar.sha1 b/plugins/discovery-gce/licenses/httpclient-4.5.2.jar.sha1 deleted file mode 100644 index 6937112a09fb6..0000000000000 --- a/plugins/discovery-gce/licenses/httpclient-4.5.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -733db77aa8d9b2d68015189df76ab06304406e50 \ No newline at end of file diff --git a/plugins/discovery-gce/licenses/httpclient-4.5.7.jar.sha1 b/plugins/discovery-gce/licenses/httpclient-4.5.7.jar.sha1 new file mode 100644 index 0000000000000..a8b7cc0d994d3 --- /dev/null +++ b/plugins/discovery-gce/licenses/httpclient-4.5.7.jar.sha1 @@ -0,0 +1 @@ +dda059f4908e1b548b7ba68d81a3b05897f27cb0 \ No newline at end of file diff --git a/plugins/discovery-gce/licenses/httpcore-4.4.11.jar.sha1 b/plugins/discovery-gce/licenses/httpcore-4.4.11.jar.sha1 new file mode 100644 index 0000000000000..6d64372bfccd8 --- /dev/null +++ b/plugins/discovery-gce/licenses/httpcore-4.4.11.jar.sha1 @@ -0,0 +1 @@ +de748cf874e4e193b42eceea9fe5574fabb9d4df \ No newline at end of file diff --git a/plugins/discovery-gce/licenses/httpcore-4.4.5.jar.sha1 b/plugins/discovery-gce/licenses/httpcore-4.4.5.jar.sha1 deleted file mode 100644 index 581726601745b..0000000000000 --- a/plugins/discovery-gce/licenses/httpcore-4.4.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e7501a1b34325abb00d17dde96150604a0658b54 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/commons-codec-1.10.jar.sha1 b/plugins/ingest-attachment/licenses/commons-codec-1.10.jar.sha1 deleted file mode 100644 index 3fe8682a1b0f9..0000000000000 --- a/plugins/ingest-attachment/licenses/commons-codec-1.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4b95f4897fa13f2cd904aee711aeafc0c5295cd8 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/commons-codec-1.11.jar.sha1 b/plugins/ingest-attachment/licenses/commons-codec-1.11.jar.sha1 new file mode 100644 index 0000000000000..b08f71a5babf0 --- /dev/null +++ b/plugins/ingest-attachment/licenses/commons-codec-1.11.jar.sha1 @@ -0,0 +1 @@ +3acb4705652e16236558f0f4f2192cc33c3bd189 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/commons-codec-1.10.jar.sha1 b/plugins/repository-gcs/licenses/commons-codec-1.10.jar.sha1 deleted file mode 100644 index 3fe8682a1b0f9..0000000000000 --- a/plugins/repository-gcs/licenses/commons-codec-1.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4b95f4897fa13f2cd904aee711aeafc0c5295cd8 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/commons-codec-1.11.jar.sha1 b/plugins/repository-gcs/licenses/commons-codec-1.11.jar.sha1 new file mode 100644 index 0000000000000..b08f71a5babf0 --- /dev/null +++ b/plugins/repository-gcs/licenses/commons-codec-1.11.jar.sha1 @@ -0,0 +1 @@ +3acb4705652e16236558f0f4f2192cc33c3bd189 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/httpclient-4.5.2.jar.sha1 b/plugins/repository-gcs/licenses/httpclient-4.5.2.jar.sha1 deleted file mode 100644 index 6937112a09fb6..0000000000000 --- a/plugins/repository-gcs/licenses/httpclient-4.5.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -733db77aa8d9b2d68015189df76ab06304406e50 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/httpclient-4.5.7.jar.sha1 b/plugins/repository-gcs/licenses/httpclient-4.5.7.jar.sha1 new file mode 100644 index 0000000000000..a8b7cc0d994d3 --- /dev/null +++ b/plugins/repository-gcs/licenses/httpclient-4.5.7.jar.sha1 @@ -0,0 +1 @@ +dda059f4908e1b548b7ba68d81a3b05897f27cb0 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/httpcore-4.4.11.jar.sha1 b/plugins/repository-gcs/licenses/httpcore-4.4.11.jar.sha1 new file mode 100644 index 0000000000000..6d64372bfccd8 --- /dev/null +++ b/plugins/repository-gcs/licenses/httpcore-4.4.11.jar.sha1 @@ -0,0 +1 @@ +de748cf874e4e193b42eceea9fe5574fabb9d4df \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/httpcore-4.4.5.jar.sha1 b/plugins/repository-gcs/licenses/httpcore-4.4.5.jar.sha1 deleted file mode 100644 index 581726601745b..0000000000000 --- a/plugins/repository-gcs/licenses/httpcore-4.4.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e7501a1b34325abb00d17dde96150604a0658b54 \ No newline at end of file diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index ac22fa389c614..34323fb930fce 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -52,7 +52,7 @@ dependencies { compile 'com.google.protobuf:protobuf-java:2.5.0' compile 'commons-logging:commons-logging:1.1.3' compile 'commons-cli:commons-cli:1.2' - compile 'commons-codec:commons-codec:1.10' + compile "commons-codec:commons-codec:${versions.commonscodec}" compile 'commons-collections:commons-collections:3.2.2' compile 'commons-configuration:commons-configuration:1.6' compile 'commons-io:commons-io:2.4' diff --git a/plugins/repository-hdfs/licenses/commons-codec-1.10.jar.sha1 b/plugins/repository-hdfs/licenses/commons-codec-1.10.jar.sha1 deleted file mode 100644 index 3fe8682a1b0f9..0000000000000 --- a/plugins/repository-hdfs/licenses/commons-codec-1.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4b95f4897fa13f2cd904aee711aeafc0c5295cd8 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/commons-codec-1.11.jar.sha1 b/plugins/repository-hdfs/licenses/commons-codec-1.11.jar.sha1 new file mode 100644 index 0000000000000..b08f71a5babf0 --- /dev/null +++ b/plugins/repository-hdfs/licenses/commons-codec-1.11.jar.sha1 @@ -0,0 +1 @@ +3acb4705652e16236558f0f4f2192cc33c3bd189 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/commons-codec-1.10.jar.sha1 b/plugins/repository-s3/licenses/commons-codec-1.10.jar.sha1 deleted file mode 100644 index 3fe8682a1b0f9..0000000000000 --- a/plugins/repository-s3/licenses/commons-codec-1.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4b95f4897fa13f2cd904aee711aeafc0c5295cd8 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/commons-codec-1.11.jar.sha1 b/plugins/repository-s3/licenses/commons-codec-1.11.jar.sha1 new file mode 100644 index 0000000000000..b08f71a5babf0 --- /dev/null +++ b/plugins/repository-s3/licenses/commons-codec-1.11.jar.sha1 @@ -0,0 +1 @@ +3acb4705652e16236558f0f4f2192cc33c3bd189 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/httpclient-4.5.2.jar.sha1 b/plugins/repository-s3/licenses/httpclient-4.5.2.jar.sha1 deleted file mode 100644 index 6937112a09fb6..0000000000000 --- a/plugins/repository-s3/licenses/httpclient-4.5.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -733db77aa8d9b2d68015189df76ab06304406e50 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/httpclient-4.5.7.jar.sha1 b/plugins/repository-s3/licenses/httpclient-4.5.7.jar.sha1 new file mode 100644 index 0000000000000..a8b7cc0d994d3 --- /dev/null +++ b/plugins/repository-s3/licenses/httpclient-4.5.7.jar.sha1 @@ -0,0 +1 @@ +dda059f4908e1b548b7ba68d81a3b05897f27cb0 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/httpcore-4.4.11.jar.sha1 b/plugins/repository-s3/licenses/httpcore-4.4.11.jar.sha1 new file mode 100644 index 0000000000000..6d64372bfccd8 --- /dev/null +++ b/plugins/repository-s3/licenses/httpcore-4.4.11.jar.sha1 @@ -0,0 +1 @@ +de748cf874e4e193b42eceea9fe5574fabb9d4df \ No newline at end of file diff --git a/plugins/repository-s3/licenses/httpcore-4.4.5.jar.sha1 b/plugins/repository-s3/licenses/httpcore-4.4.5.jar.sha1 deleted file mode 100644 index 581726601745b..0000000000000 --- a/plugins/repository-s3/licenses/httpcore-4.4.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e7501a1b34325abb00d17dde96150604a0658b54 \ No newline at end of file diff --git a/x-pack/plugin/core/licenses/commons-codec-1.10.jar.sha1 b/x-pack/plugin/core/licenses/commons-codec-1.10.jar.sha1 deleted file mode 100644 index 3fe8682a1b0f9..0000000000000 --- a/x-pack/plugin/core/licenses/commons-codec-1.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4b95f4897fa13f2cd904aee711aeafc0c5295cd8 \ No newline at end of file diff --git a/x-pack/plugin/core/licenses/commons-codec-1.11.jar.sha1 b/x-pack/plugin/core/licenses/commons-codec-1.11.jar.sha1 new file mode 100644 index 0000000000000..b08f71a5babf0 --- /dev/null +++ b/x-pack/plugin/core/licenses/commons-codec-1.11.jar.sha1 @@ -0,0 +1 @@ +3acb4705652e16236558f0f4f2192cc33c3bd189 \ No newline at end of file diff --git a/x-pack/plugin/core/licenses/httpasyncclient-4.1.2.jar.sha1 b/x-pack/plugin/core/licenses/httpasyncclient-4.1.2.jar.sha1 deleted file mode 100644 index 065ed920a1773..0000000000000 --- a/x-pack/plugin/core/licenses/httpasyncclient-4.1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -95aa3e6fb520191a0970a73cf09f62948ee614be \ No newline at end of file diff --git a/x-pack/plugin/core/licenses/httpasyncclient-4.1.4.jar.sha1 b/x-pack/plugin/core/licenses/httpasyncclient-4.1.4.jar.sha1 new file mode 100644 index 0000000000000..8360ab45c7ab3 --- /dev/null +++ b/x-pack/plugin/core/licenses/httpasyncclient-4.1.4.jar.sha1 @@ -0,0 +1 @@ +f3a3240681faae3fa46b573a4c7e50cec9db0d86 \ No newline at end of file diff --git a/x-pack/plugin/core/licenses/httpclient-4.5.2.jar.sha1 b/x-pack/plugin/core/licenses/httpclient-4.5.2.jar.sha1 deleted file mode 100644 index 6937112a09fb6..0000000000000 --- a/x-pack/plugin/core/licenses/httpclient-4.5.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -733db77aa8d9b2d68015189df76ab06304406e50 \ No newline at end of file diff --git a/x-pack/plugin/core/licenses/httpclient-4.5.7.jar.sha1 b/x-pack/plugin/core/licenses/httpclient-4.5.7.jar.sha1 new file mode 100644 index 0000000000000..a8b7cc0d994d3 --- /dev/null +++ b/x-pack/plugin/core/licenses/httpclient-4.5.7.jar.sha1 @@ -0,0 +1 @@ +dda059f4908e1b548b7ba68d81a3b05897f27cb0 \ No newline at end of file diff --git a/x-pack/plugin/core/licenses/httpcore-4.4.11.jar.sha1 b/x-pack/plugin/core/licenses/httpcore-4.4.11.jar.sha1 new file mode 100644 index 0000000000000..6d64372bfccd8 --- /dev/null +++ b/x-pack/plugin/core/licenses/httpcore-4.4.11.jar.sha1 @@ -0,0 +1 @@ +de748cf874e4e193b42eceea9fe5574fabb9d4df \ No newline at end of file diff --git a/x-pack/plugin/core/licenses/httpcore-4.4.5.jar.sha1 b/x-pack/plugin/core/licenses/httpcore-4.4.5.jar.sha1 deleted file mode 100644 index 581726601745b..0000000000000 --- a/x-pack/plugin/core/licenses/httpcore-4.4.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e7501a1b34325abb00d17dde96150604a0658b54 \ No newline at end of file diff --git a/x-pack/plugin/core/licenses/httpcore-nio-4.4.11.jar.sha1 b/x-pack/plugin/core/licenses/httpcore-nio-4.4.11.jar.sha1 new file mode 100644 index 0000000000000..9e8777cb3da1c --- /dev/null +++ b/x-pack/plugin/core/licenses/httpcore-nio-4.4.11.jar.sha1 @@ -0,0 +1 @@ +7d0a97d01d39cff9aa3e6db81f21fddb2435f4e6 \ No newline at end of file diff --git a/x-pack/plugin/core/licenses/httpcore-nio-4.4.5.jar.sha1 b/x-pack/plugin/core/licenses/httpcore-nio-4.4.5.jar.sha1 deleted file mode 100644 index d6a80bf100de3..0000000000000 --- a/x-pack/plugin/core/licenses/httpcore-nio-4.4.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f4be009e7505f6ceddf21e7960c759f413f15056 \ No newline at end of file diff --git a/x-pack/plugin/security/licenses/httpclient-cache-4.5.2.jar.sha1 b/x-pack/plugin/security/licenses/httpclient-cache-4.5.2.jar.sha1 deleted file mode 100644 index 75fbd3009da8e..0000000000000 --- a/x-pack/plugin/security/licenses/httpclient-cache-4.5.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bd50ea83908dbf2f387a333216e66d2f0c5079bd \ No newline at end of file diff --git a/x-pack/plugin/security/licenses/httpclient-cache-4.5.7.jar.sha1 b/x-pack/plugin/security/licenses/httpclient-cache-4.5.7.jar.sha1 new file mode 100644 index 0000000000000..b121bd654212b --- /dev/null +++ b/x-pack/plugin/security/licenses/httpclient-cache-4.5.7.jar.sha1 @@ -0,0 +1 @@ +c13a0ce27c17831e5e5be6c751842006dcecb270 \ No newline at end of file diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java index 10fb8889fae33..654bc6b757dde 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java @@ -47,6 +47,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; @@ -162,7 +163,9 @@ private void setWhitelistAutomaton(List whiteListedHosts) { } public HttpResponse execute(HttpRequest request) throws IOException { - URI uri = createURI(request); + Tuple tuple = createURI(request); + final URI uri = tuple.v2(); + final HttpHost httpHost = tuple.v1(); HttpRequestBase internalRequest; if (request.method == HttpMethod.HEAD) { @@ -212,7 +215,7 @@ public HttpResponse execute(HttpRequest request) throws IOException { // preemptive auth, no need to wait for a 401 first AuthCache authCache = new BasicAuthCache(); BasicScheme basicAuth = new BasicScheme(); - authCache.put(new HttpHost(request.host, request.port, request.scheme.scheme()), basicAuth); + authCache.put(httpHost, basicAuth); localContext.setAuthCache(authCache); } @@ -233,7 +236,7 @@ public HttpResponse execute(HttpRequest request) throws IOException { internalRequest.setConfig(config.build()); - try (CloseableHttpResponse response = SocketAccess.doPrivileged(() -> client.execute(internalRequest, localContext))) { + try (CloseableHttpResponse response = SocketAccess.doPrivileged(() -> client.execute(httpHost, internalRequest, localContext))) { // headers Header[] headers = response.getAllHeaders(); Map responseHeaders = new HashMap<>(headers.length); @@ -310,7 +313,7 @@ private HttpProxy getProxyFromSettings(Settings settings) { return HttpProxy.NO_PROXY; } - private URI createURI(HttpRequest request) { + private Tuple createURI(HttpRequest request) { // this could be really simple, as the apache http client has a UriBuilder class, however this class is always doing // url path escaping, and we have done this already, so this would result in double escaping try { @@ -320,7 +323,23 @@ private URI createURI(HttpRequest request) { URI uri = URIUtils.createURI(request.scheme.scheme(), request.host, request.port, request.path, Strings.isNullOrEmpty(format) ? null : format, null); - return uri; + if (uri.isAbsolute() == false) { + throw new IllegalStateException("URI [" + uri.toASCIIString() + "] must be absolute"); + } + final HttpHost httpHost = URIUtils.extractHost(uri); + // what a mess that we need to do this to workaround https://issues.apache.org/jira/browse/HTTPCLIENT-1968 + // in some cases the HttpClient will re-write the URI which drops the escaping for + // slashes within a path. This rewriting is done to obtain a relative URI when + // a proxy is not being used. To avoid this we can handle making it relative ourselves + if (request.path != null && request.path.contains("%2F")) { + final boolean isUsingProxy = (request.proxy != null && request.proxy.equals(HttpProxy.NO_PROXY) == false) || + HttpProxy.NO_PROXY.equals(settingsProxy) == false; + if (isUsingProxy == false) { + // we need a relative uri + uri = URIUtils.createURI(null, null, -1, request.path, Strings.isNullOrEmpty(format) ? null : format, null); + } + } + return new Tuple<>(httpHost, uri); } catch (URISyntaxException e) { throw new IllegalArgumentException(e); } From b88bdfe958ccedb96f624bbd796ab10c5e5f70cb Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Wed, 30 Jan 2019 15:40:49 -0700 Subject: [PATCH 074/100] Add dispatching to `HandledTransportAction` (#38050) This commit allows implementors of the `HandledTransportAction` to specify what thread the action should be executed on. The motivation for this commit is that certain CCR requests should be performed on the generic threadpool. --- .../support/HandledTransportAction.java | 13 ++++++- .../ClearCcrRestoreSessionAction.java | 12 ++---- .../GetCcrRestoreFileChunkAction.java | 39 +++++++------------ 3 files changed, 30 insertions(+), 34 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java b/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java index 0b35bc8fb89d6..c0bc0af839967 100644 --- a/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java @@ -45,6 +45,11 @@ protected HandledTransportAction(String actionName, TransportService transportSe this(actionName, true, transportService, actionFilters, requestReader); } + protected HandledTransportAction(String actionName, TransportService transportService, + ActionFilters actionFilters, Writeable.Reader requestReader, String executor) { + this(actionName, true, transportService, actionFilters, requestReader, executor); + } + protected HandledTransportAction(String actionName, boolean canTripCircuitBreaker, TransportService transportService, ActionFilters actionFilters, Supplier request) { super(actionName, actionFilters, transportService.getTaskManager()); @@ -55,8 +60,14 @@ protected HandledTransportAction(String actionName, boolean canTripCircuitBreake protected HandledTransportAction(String actionName, boolean canTripCircuitBreaker, TransportService transportService, ActionFilters actionFilters, Writeable.Reader requestReader) { + this(actionName, canTripCircuitBreaker, transportService, actionFilters, requestReader, ThreadPool.Names.SAME); + } + + protected HandledTransportAction(String actionName, boolean canTripCircuitBreaker, + TransportService transportService, ActionFilters actionFilters, + Writeable.Reader requestReader, String executor) { super(actionName, actionFilters, transportService.getTaskManager()); - transportService.registerRequestHandler(actionName, ThreadPool.Names.SAME, false, canTripCircuitBreaker, requestReader, + transportService.registerRequestHandler(actionName, executor, false, canTripCircuitBreaker, requestReader, new TransportHandler()); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionAction.java index 81cde2984f500..317890edb4206 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionAction.java @@ -43,26 +43,20 @@ public static class TransportDeleteCcrRestoreSessionAction extends HandledTransportAction { private final CcrRestoreSourceService ccrRestoreService; - private final ThreadPool threadPool; @Inject public TransportDeleteCcrRestoreSessionAction(ActionFilters actionFilters, TransportService transportService, CcrRestoreSourceService ccrRestoreService) { - super(NAME, transportService, actionFilters, ClearCcrRestoreSessionRequest::new); + super(NAME, transportService, actionFilters, ClearCcrRestoreSessionRequest::new, ThreadPool.Names.GENERIC); TransportActionProxy.registerProxyAction(transportService, NAME, ClearCcrRestoreSessionResponse::new); this.ccrRestoreService = ccrRestoreService; - this.threadPool = transportService.getThreadPool(); } @Override protected void doExecute(Task task, ClearCcrRestoreSessionRequest request, ActionListener listener) { - // TODO: Currently blocking actions might occur in the session closed callbacks. This dispatch - // may be unnecessary when we remove these callbacks. - threadPool.generic().execute(() -> { - ccrRestoreService.closeSession(request.getSessionUUID()); - listener.onResponse(new ClearCcrRestoreSessionResponse()); - }); + ccrRestoreService.closeSession(request.getSessionUUID()); + listener.onResponse(new ClearCcrRestoreSessionResponse()); } } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkAction.java index 3f473f25c2411..cf8d2e5c55f48 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkAction.java @@ -19,7 +19,6 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.ByteArray; -import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportActionProxy; @@ -58,7 +57,7 @@ public static class TransportGetCcrRestoreFileChunkAction @Inject public TransportGetCcrRestoreFileChunkAction(BigArrays bigArrays, TransportService transportService, ActionFilters actionFilters, CcrRestoreSourceService restoreSourceService) { - super(NAME, transportService, actionFilters, GetCcrRestoreFileChunkRequest::new); + super(NAME, transportService, actionFilters, GetCcrRestoreFileChunkRequest::new, ThreadPool.Names.GENERIC); TransportActionProxy.registerProxyAction(transportService, NAME, GetCcrRestoreFileChunkResponse::new); this.threadPool = transportService.getThreadPool(); this.restoreSourceService = restoreSourceService; @@ -68,29 +67,21 @@ public TransportGetCcrRestoreFileChunkAction(BigArrays bigArrays, TransportServi @Override protected void doExecute(Task task, GetCcrRestoreFileChunkRequest request, ActionListener listener) { - threadPool.generic().execute(new AbstractRunnable() { - @Override - public void onFailure(Exception e) { - listener.onFailure(e); + int bytesRequested = request.getSize(); + ByteArray array = bigArrays.newByteArray(bytesRequested, false); + String fileName = request.getFileName(); + String sessionUUID = request.getSessionUUID(); + // This is currently safe to do because calling `onResponse` will serialize the bytes to the network layer data + // structure on the same thread. So the bytes will be copied before the reference is released. + try (ReleasablePagedBytesReference reference = new ReleasablePagedBytesReference(array, bytesRequested, array)) { + try (CcrRestoreSourceService.SessionReader sessionReader = restoreSourceService.getSessionReader(sessionUUID)) { + long offsetAfterRead = sessionReader.readFileBytes(fileName, reference); + long offsetBeforeRead = offsetAfterRead - reference.length(); + listener.onResponse(new GetCcrRestoreFileChunkResponse(offsetBeforeRead, reference)); } - - @Override - protected void doRun() throws Exception { - int bytesRequested = request.getSize(); - ByteArray array = bigArrays.newByteArray(bytesRequested, false); - String fileName = request.getFileName(); - String sessionUUID = request.getSessionUUID(); - // This is currently safe to do because calling `onResponse` will serialize the bytes to the network layer data - // structure on the same thread. So the bytes will be copied before the reference is released. - try (ReleasablePagedBytesReference reference = new ReleasablePagedBytesReference(array, bytesRequested, array)) { - try (CcrRestoreSourceService.SessionReader sessionReader = restoreSourceService.getSessionReader(sessionUUID)) { - long offsetAfterRead = sessionReader.readFileBytes(fileName, reference); - long offsetBeforeRead = offsetAfterRead - reference.length(); - listener.onResponse(new GetCcrRestoreFileChunkResponse(offsetBeforeRead, reference)); - } - } - } - }); + } catch (IOException e) { + listener.onFailure(e); + } } } From 7c738fd2414a5a11954d201a4af49307472b99fc Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Wed, 30 Jan 2019 15:09:17 -0800 Subject: [PATCH 075/100] Skip Shrink when numberOfShards not changed (#37953) Previously, ShrinkAction would fail if it was executed on an index that had the same number of shards as the target shrunken number. This PR introduced a new BranchingStep that is used inside of ShrinkAction to branch which step to move to next, depending on the shard values. So no shrink will occur if the shard count is unchanged. --- .../documentation/ILMDocumentationIT.java | 4 +- .../core/indexlifecycle/BranchingStep.java | 114 +++++++++++++++ .../core/indexlifecycle/ShrinkAction.java | 10 +- .../xpack/core/indexlifecycle/Step.java | 2 +- .../indexlifecycle/BranchingStepTests.java | 85 +++++++++++ .../indexlifecycle/ShrinkActionTests.java | 133 ++++++++++++++---- .../TimeSeriesLifecycleActionsIT.java | 18 +++ .../ExecuteStepsUpdateTask.java | 16 ++- .../ExecuteStepsUpdateTaskTests.java | 2 +- 9 files changed, 346 insertions(+), 38 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/BranchingStep.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/BranchingStepTests.java diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/ILMDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/ILMDocumentationIT.java index 5ccb0c8393304..db9df0ac24c78 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/ILMDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/ILMDocumentationIT.java @@ -591,7 +591,7 @@ public void testRetryPolicy() throws Exception { { Map phases = new HashMap<>(); Map warmActions = new HashMap<>(); - warmActions.put(ShrinkAction.NAME, new ShrinkAction(1)); + warmActions.put(ShrinkAction.NAME, new ShrinkAction(3)); phases.put("warm", new Phase("warm", TimeValue.ZERO, warmActions)); LifecyclePolicy policy = new LifecyclePolicy("my_policy", @@ -602,7 +602,7 @@ public void testRetryPolicy() throws Exception { CreateIndexRequest createIndexRequest = new CreateIndexRequest("my_index") .settings(Settings.builder() - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2) .put("index.lifecycle.name", "my_policy") .build()); client.indices().create(createIndexRequest, RequestOptions.DEFAULT); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/BranchingStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/BranchingStep.java new file mode 100644 index 0000000000000..2514492520200 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/BranchingStep.java @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.indexlifecycle; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.util.SetOnce; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.index.Index; + +import java.util.Objects; +import java.util.function.BiPredicate; + +/** + * This step changes its {@link #getNextStepKey()} depending on the + * outcome of a defined predicate. It performs no changes to the + * cluster state. + */ +public class BranchingStep extends ClusterStateActionStep { + public static final String NAME = "branch"; + + private static final Logger logger = LogManager.getLogger(BranchingStep.class); + + private StepKey nextStepKeyOnFalse; + private StepKey nextStepKeyOnTrue; + private BiPredicate predicate; + private SetOnce predicateValue; + + /** + * {@link BranchingStep} is a step whose next step is based on + * the return value of a specific predicate. + * + * @param key the step's key + * @param nextStepKeyOnFalse the key of the step to run if predicate returns false + * @param nextStepKeyOnTrue the key of the step to run if predicate returns true + * @param predicate the condition to check when deciding which step to run next + */ + public BranchingStep(StepKey key, StepKey nextStepKeyOnFalse, StepKey nextStepKeyOnTrue, BiPredicate predicate) { + // super.nextStepKey is set to null since it is not used by this step + super(key, null); + this.nextStepKeyOnFalse = nextStepKeyOnFalse; + this.nextStepKeyOnTrue = nextStepKeyOnTrue; + this.predicate = predicate; + this.predicateValue = new SetOnce<>(); + } + + @Override + public ClusterState performAction(Index index, ClusterState clusterState) { + IndexMetaData indexMetaData = clusterState.metaData().index(index); + if (indexMetaData == null) { + // Index must have been since deleted, ignore it + logger.debug("[{}] lifecycle action for index [{}] executed but index no longer exists", getKey().getAction(), index.getName()); + return clusterState; + } + predicateValue.set(predicate.test(index, clusterState)); + return clusterState; + } + + /** + * This method returns the next step to execute based on the predicate. If + * the predicate returned true, then nextStepKeyOnTrue is the key of the + * next step to run, otherwise nextStepKeyOnFalse is. + * + * throws {@link UnsupportedOperationException} if performAction was not called yet + * + * @return next step to execute + */ + @Override + public final StepKey getNextStepKey() { + if (predicateValue.get() == null) { + throw new IllegalStateException("Cannot call getNextStepKey before performAction"); + } + return predicateValue.get() ? nextStepKeyOnTrue : nextStepKeyOnFalse; + } + + /** + * @return the next step if {@code predicate} is false + */ + final StepKey getNextStepKeyOnFalse() { + return nextStepKeyOnFalse; + } + + /** + * @return the next step if {@code predicate} is true + */ + final StepKey getNextStepKeyOnTrue() { + return nextStepKeyOnTrue; + } + + public final BiPredicate getPredicate() { + return predicate; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + BranchingStep that = (BranchingStep) o; + return super.equals(o) + && Objects.equals(nextStepKeyOnFalse, that.nextStepKeyOnFalse) + && Objects.equals(nextStepKeyOnTrue, that.nextStepKeyOnTrue); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), nextStepKeyOnFalse, nextStepKeyOnTrue); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkAction.java index a79383c24de8b..51f24e6d65254 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkAction.java @@ -85,6 +85,7 @@ public boolean isSafeAction() { public List toSteps(Client client, String phase, Step.StepKey nextStepKey) { Settings readOnlySettings = Settings.builder().put(IndexMetaData.SETTING_BLOCKS_WRITE, true).build(); + StepKey branchingKey = new StepKey(phase, NAME, BranchingStep.NAME); StepKey readOnlyKey = new StepKey(phase, NAME, ReadOnlyAction.NAME); StepKey setSingleNodeKey = new StepKey(phase, NAME, SetSingleNodeAllocateStep.NAME); StepKey allocationRoutedKey = new StepKey(phase, NAME, CheckShrinkReadyStep.NAME); @@ -94,6 +95,8 @@ public List toSteps(Client client, String phase, Step.StepKey nextStepKey) StepKey aliasKey = new StepKey(phase, NAME, ShrinkSetAliasStep.NAME); StepKey isShrunkIndexKey = new StepKey(phase, NAME, ShrunkenIndexCheckStep.NAME); + BranchingStep conditionalSkipShrinkStep = new BranchingStep(branchingKey, readOnlyKey, nextStepKey, + (index, clusterState) -> clusterState.getMetaData().index(index).getNumberOfShards() == numberOfShards); UpdateSettingsStep readOnlyStep = new UpdateSettingsStep(readOnlyKey, setSingleNodeKey, client, readOnlySettings); SetSingleNodeAllocateStep setSingleNodeStep = new SetSingleNodeAllocateStep(setSingleNodeKey, allocationRoutedKey, client); CheckShrinkReadyStep checkShrinkReadyStep = new CheckShrinkReadyStep(allocationRoutedKey, shrinkKey); @@ -102,12 +105,13 @@ public List toSteps(Client client, String phase, Step.StepKey nextStepKey) CopyExecutionStateStep copyMetadata = new CopyExecutionStateStep(copyMetadataKey, aliasKey, SHRUNKEN_INDEX_PREFIX); ShrinkSetAliasStep aliasSwapAndDelete = new ShrinkSetAliasStep(aliasKey, isShrunkIndexKey, client, SHRUNKEN_INDEX_PREFIX); ShrunkenIndexCheckStep waitOnShrinkTakeover = new ShrunkenIndexCheckStep(isShrunkIndexKey, nextStepKey, SHRUNKEN_INDEX_PREFIX); - return Arrays.asList(readOnlyStep, setSingleNodeStep, checkShrinkReadyStep, shrink, allocated, copyMetadata, - aliasSwapAndDelete, waitOnShrinkTakeover); + return Arrays.asList(conditionalSkipShrinkStep, readOnlyStep, setSingleNodeStep, checkShrinkReadyStep, shrink, allocated, + copyMetadata, aliasSwapAndDelete, waitOnShrinkTakeover); } @Override public List toStepKeys(String phase) { + StepKey conditionalSkipKey = new StepKey(phase, NAME, BranchingStep.NAME); StepKey readOnlyKey = new StepKey(phase, NAME, ReadOnlyAction.NAME); StepKey setSingleNodeKey = new StepKey(phase, NAME, SetSingleNodeAllocateStep.NAME); StepKey checkShrinkReadyKey = new StepKey(phase, NAME, CheckShrinkReadyStep.NAME); @@ -116,7 +120,7 @@ public List toStepKeys(String phase) { StepKey copyMetadataKey = new StepKey(phase, NAME, CopyExecutionStateStep.NAME); StepKey aliasKey = new StepKey(phase, NAME, ShrinkSetAliasStep.NAME); StepKey isShrunkIndexKey = new StepKey(phase, NAME, ShrunkenIndexCheckStep.NAME); - return Arrays.asList(readOnlyKey, setSingleNodeKey, checkShrinkReadyKey, shrinkKey, enoughShardsKey, + return Arrays.asList(conditionalSkipKey, readOnlyKey, setSingleNodeKey, checkShrinkReadyKey, shrinkKey, enoughShardsKey, copyMetadataKey, aliasKey, isShrunkIndexKey); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/Step.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/Step.java index 5f24ab29d0284..4917a2aafd433 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/Step.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/Step.java @@ -34,7 +34,7 @@ public final StepKey getKey() { return key; } - public final StepKey getNextStepKey() { + public StepKey getNextStepKey() { return nextStepKey; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/BranchingStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/BranchingStepTests.java new file mode 100644 index 0000000000000..6c354b79203cd --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/BranchingStepTests.java @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.indexlifecycle; + +import org.apache.lucene.util.SetOnce; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.index.Index; +import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey; + +import java.util.function.BiPredicate; + +import static org.hamcrest.Matchers.equalTo; + +public class BranchingStepTests extends AbstractStepTestCase { + + public void testPredicateNextStepChange() { + String indexName = randomAlphaOfLength(5); + ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metaData(MetaData.builder() + .put(IndexMetaData.builder(indexName).settings(settings(Version.CURRENT)) + .numberOfShards(1).numberOfReplicas(0))).build(); + StepKey stepKey = new StepKey(randomAlphaOfLength(5), randomAlphaOfLength(5), BranchingStep.NAME); + StepKey nextStepKey = new StepKey(randomAlphaOfLength(6), randomAlphaOfLength(6), BranchingStep.NAME); + StepKey nextSkipKey = new StepKey(randomAlphaOfLength(7), randomAlphaOfLength(7), BranchingStep.NAME); + { + BranchingStep step = new BranchingStep(stepKey, nextStepKey, nextSkipKey, (i, c) -> true); + expectThrows(IllegalStateException.class, step::getNextStepKey); + step.performAction(state.metaData().index(indexName).getIndex(), state); + assertThat(step.getNextStepKey(), equalTo(step.getNextStepKeyOnTrue())); + expectThrows(SetOnce.AlreadySetException.class, () -> step.performAction(state.metaData().index(indexName).getIndex(), state)); + } + { + BranchingStep step = new BranchingStep(stepKey, nextStepKey, nextSkipKey, (i, c) -> false); + expectThrows(IllegalStateException.class, step::getNextStepKey); + step.performAction(state.metaData().index(indexName).getIndex(), state); + assertThat(step.getNextStepKey(), equalTo(step.getNextStepKeyOnFalse())); + expectThrows(SetOnce.AlreadySetException.class, () -> step.performAction(state.metaData().index(indexName).getIndex(), state)); + } + } + + @Override + public BranchingStep createRandomInstance() { + StepKey stepKey = new StepKey(randomAlphaOfLength(5), randomAlphaOfLength(5), BranchingStep.NAME); + StepKey nextStepKey = new StepKey(randomAlphaOfLength(6), randomAlphaOfLength(6), BranchingStep.NAME); + StepKey nextSkipKey = new StepKey(randomAlphaOfLength(7), randomAlphaOfLength(7), BranchingStep.NAME); + return new BranchingStep(stepKey, nextStepKey, nextSkipKey, (i, c) -> randomBoolean()); + } + + @Override + public BranchingStep mutateInstance(BranchingStep instance) { + StepKey key = instance.getKey(); + StepKey nextStepKey = instance.getNextStepKeyOnFalse(); + StepKey nextSkipStepKey = instance.getNextStepKeyOnTrue(); + BiPredicate predicate = instance.getPredicate(); + + switch (between(0, 2)) { + case 0: + key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5)); + break; + case 1: + nextStepKey = new StepKey(nextStepKey.getPhase(), nextStepKey.getAction(), nextStepKey.getName() + randomAlphaOfLength(5)); + break; + case 2: + nextSkipStepKey = new StepKey(nextSkipStepKey.getPhase(), nextSkipStepKey.getAction(), + nextSkipStepKey.getName() + randomAlphaOfLength(5)); + break; + default: + throw new AssertionError("Illegal randomisation branch"); + } + + return new BranchingStep(key, nextStepKey, nextSkipStepKey, predicate); + } + + @Override + public BranchingStep copyInstance(BranchingStep instance) { + return new BranchingStep(instance.getKey(), instance.getNextStepKeyOnFalse(), instance.getNextStepKeyOnTrue(), + instance.getPredicate()); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkActionTests.java index 658f8bef6d47b..be512c87d8548 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkActionTests.java @@ -5,12 +5,18 @@ */ package org.elasticsearch.xpack.core.indexlifecycle; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey; import java.io.IOException; +import java.util.Collections; import java.util.List; import static org.hamcrest.Matchers.equalTo; @@ -46,59 +52,134 @@ public void testNonPositiveShardNumber() { assertThat(e.getMessage(), equalTo("[number_of_shards] must be greater than 0")); } + public void testPerformActionWithSkip() { + String lifecycleName = randomAlphaOfLengthBetween(4, 10); + int numberOfShards = randomIntBetween(1, 10); + ShrinkAction action = new ShrinkAction(numberOfShards); + String phase = randomAlphaOfLengthBetween(1, 10); + StepKey nextStepKey = new StepKey(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10), + randomAlphaOfLengthBetween(1, 10)); + List steps = action.toSteps(null, phase, nextStepKey); + BranchingStep step = ((BranchingStep) steps.get(0)); + + LifecyclePolicy policy = new LifecyclePolicy(lifecycleName, Collections.singletonMap("warm", + new Phase("warm", TimeValue.ZERO, Collections.singletonMap(action.getWriteableName(), action)))); + LifecyclePolicyMetadata policyMetadata = new LifecyclePolicyMetadata(policy, Collections.emptyMap(), + randomNonNegativeLong(), randomNonNegativeLong()); + String indexName = randomAlphaOfLength(5); + ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metaData(MetaData.builder() + .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata( + Collections.singletonMap(policyMetadata.getName(), policyMetadata), OperationMode.RUNNING)) + .put(IndexMetaData.builder(indexName).settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME, lifecycleName)) + .putCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY, + LifecycleExecutionState.builder() + .setPhase(step.getKey().getPhase()) + .setPhaseTime(0L) + .setAction(step.getKey().getAction()) + .setActionTime(0L) + .setStep(step.getKey().getName()) + .setStepTime(0L) + .build().asMap()) + .numberOfShards(numberOfShards).numberOfReplicas(0))).build(); + step.performAction(state.metaData().index(indexName).getIndex(), state); + assertThat(step.getNextStepKey(), equalTo(nextStepKey)); + } + + public void testPerformActionWithoutSkip() { + int numShards = 6; + int divisor = randomFrom(2, 3, 6); + int expectedFinalShards = numShards / divisor; + String lifecycleName = randomAlphaOfLengthBetween(4, 10); + ShrinkAction action = new ShrinkAction(expectedFinalShards); + String phase = randomAlphaOfLengthBetween(1, 10); + StepKey nextStepKey = new StepKey(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10), + randomAlphaOfLengthBetween(1, 10)); + List steps = action.toSteps(null, phase, nextStepKey); + BranchingStep step = ((BranchingStep) steps.get(0)); + + LifecyclePolicy policy = new LifecyclePolicy(lifecycleName, Collections.singletonMap("warm", + new Phase("warm", TimeValue.ZERO, Collections.singletonMap(action.getWriteableName(), action)))); + LifecyclePolicyMetadata policyMetadata = new LifecyclePolicyMetadata(policy, Collections.emptyMap(), + randomNonNegativeLong(), randomNonNegativeLong()); + String indexName = randomAlphaOfLength(5); + ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metaData(MetaData.builder() + .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata( + Collections.singletonMap(policyMetadata.getName(), policyMetadata), OperationMode.RUNNING)) + .put(IndexMetaData.builder(indexName).settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME, lifecycleName)) + .putCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY, + LifecycleExecutionState.builder() + .setPhase(step.getKey().getPhase()) + .setPhaseTime(0L) + .setAction(step.getKey().getAction()) + .setActionTime(0L) + .setStep(step.getKey().getName()) + .setStepTime(0L) + .build().asMap()) + .numberOfShards(numShards).numberOfReplicas(0))).build(); + ClusterState newState = step.performAction(state.metaData().index(indexName).getIndex(), state); + assertThat(step.getNextStepKey(), equalTo(steps.get(1).getKey())); + } + public void testToSteps() { ShrinkAction action = createTestInstance(); String phase = randomAlphaOfLengthBetween(1, 10); StepKey nextStepKey = new StepKey(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10)); List steps = action.toSteps(null, phase, nextStepKey); - assertThat(steps.size(), equalTo(8)); - StepKey expectedFirstKey = new StepKey(phase, ShrinkAction.NAME, ReadOnlyAction.NAME); - StepKey expectedSecondKey = new StepKey(phase, ShrinkAction.NAME, SetSingleNodeAllocateStep.NAME); - StepKey expectedThirdKey = new StepKey(phase, ShrinkAction.NAME, CheckShrinkReadyStep.NAME); - StepKey expectedFourthKey = new StepKey(phase, ShrinkAction.NAME, ShrinkStep.NAME); - StepKey expectedFifthKey = new StepKey(phase, ShrinkAction.NAME, ShrunkShardsAllocatedStep.NAME); - StepKey expectedSixthKey = new StepKey(phase, ShrinkAction.NAME, CopyExecutionStateStep.NAME); - StepKey expectedSeventhKey = new StepKey(phase, ShrinkAction.NAME, ShrinkSetAliasStep.NAME); - StepKey expectedEighthKey = new StepKey(phase, ShrinkAction.NAME, ShrunkenIndexCheckStep.NAME); - - assertTrue(steps.get(0) instanceof UpdateSettingsStep); + assertThat(steps.size(), equalTo(9)); + StepKey expectedFirstKey = new StepKey(phase, ShrinkAction.NAME, BranchingStep.NAME); + StepKey expectedSecondKey = new StepKey(phase, ShrinkAction.NAME, ReadOnlyAction.NAME); + StepKey expectedThirdKey = new StepKey(phase, ShrinkAction.NAME, SetSingleNodeAllocateStep.NAME); + StepKey expectedFourthKey = new StepKey(phase, ShrinkAction.NAME, CheckShrinkReadyStep.NAME); + StepKey expectedFifthKey = new StepKey(phase, ShrinkAction.NAME, ShrinkStep.NAME); + StepKey expectedSixthKey = new StepKey(phase, ShrinkAction.NAME, ShrunkShardsAllocatedStep.NAME); + StepKey expectedSeventhKey = new StepKey(phase, ShrinkAction.NAME, CopyExecutionStateStep.NAME); + StepKey expectedEighthKey = new StepKey(phase, ShrinkAction.NAME, ShrinkSetAliasStep.NAME); + StepKey expectedNinthKey = new StepKey(phase, ShrinkAction.NAME, ShrunkenIndexCheckStep.NAME); + + assertTrue(steps.get(0) instanceof BranchingStep); assertThat(steps.get(0).getKey(), equalTo(expectedFirstKey)); - assertThat(steps.get(0).getNextStepKey(), equalTo(expectedSecondKey)); - assertTrue(IndexMetaData.INDEX_BLOCKS_WRITE_SETTING.get(((UpdateSettingsStep)steps.get(0)).getSettings())); + expectThrows(IllegalStateException.class, () -> steps.get(0).getNextStepKey()); + assertThat(((BranchingStep) steps.get(0)).getNextStepKeyOnFalse(), equalTo(expectedSecondKey)); + assertThat(((BranchingStep) steps.get(0)).getNextStepKeyOnTrue(), equalTo(nextStepKey)); - assertTrue(steps.get(1) instanceof SetSingleNodeAllocateStep); + assertTrue(steps.get(1) instanceof UpdateSettingsStep); assertThat(steps.get(1).getKey(), equalTo(expectedSecondKey)); assertThat(steps.get(1).getNextStepKey(), equalTo(expectedThirdKey)); + assertTrue(IndexMetaData.INDEX_BLOCKS_WRITE_SETTING.get(((UpdateSettingsStep)steps.get(1)).getSettings())); - assertTrue(steps.get(2) instanceof CheckShrinkReadyStep); + assertTrue(steps.get(2) instanceof SetSingleNodeAllocateStep); assertThat(steps.get(2).getKey(), equalTo(expectedThirdKey)); assertThat(steps.get(2).getNextStepKey(), equalTo(expectedFourthKey)); - assertTrue(steps.get(3) instanceof ShrinkStep); + assertTrue(steps.get(3) instanceof CheckShrinkReadyStep); assertThat(steps.get(3).getKey(), equalTo(expectedFourthKey)); assertThat(steps.get(3).getNextStepKey(), equalTo(expectedFifthKey)); - assertThat(((ShrinkStep) steps.get(3)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX)); - assertTrue(steps.get(4) instanceof ShrunkShardsAllocatedStep); + assertTrue(steps.get(4) instanceof ShrinkStep); assertThat(steps.get(4).getKey(), equalTo(expectedFifthKey)); assertThat(steps.get(4).getNextStepKey(), equalTo(expectedSixthKey)); - assertThat(((ShrunkShardsAllocatedStep) steps.get(4)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX)); + assertThat(((ShrinkStep) steps.get(4)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX)); - assertTrue(steps.get(5) instanceof CopyExecutionStateStep); + assertTrue(steps.get(5) instanceof ShrunkShardsAllocatedStep); assertThat(steps.get(5).getKey(), equalTo(expectedSixthKey)); assertThat(steps.get(5).getNextStepKey(), equalTo(expectedSeventhKey)); - assertThat(((CopyExecutionStateStep) steps.get(5)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX)); + assertThat(((ShrunkShardsAllocatedStep) steps.get(5)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX)); - assertTrue(steps.get(6) instanceof ShrinkSetAliasStep); + assertTrue(steps.get(6) instanceof CopyExecutionStateStep); assertThat(steps.get(6).getKey(), equalTo(expectedSeventhKey)); assertThat(steps.get(6).getNextStepKey(), equalTo(expectedEighthKey)); - assertThat(((ShrinkSetAliasStep) steps.get(6)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX)); + assertThat(((CopyExecutionStateStep) steps.get(6)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX)); - assertTrue(steps.get(7) instanceof ShrunkenIndexCheckStep); + assertTrue(steps.get(7) instanceof ShrinkSetAliasStep); assertThat(steps.get(7).getKey(), equalTo(expectedEighthKey)); - assertThat(steps.get(7).getNextStepKey(), equalTo(nextStepKey)); - assertThat(((ShrunkenIndexCheckStep) steps.get(7)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX)); + assertThat(steps.get(7).getNextStepKey(), equalTo(expectedNinthKey)); + assertThat(((ShrinkSetAliasStep) steps.get(7)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX)); + + assertTrue(steps.get(8) instanceof ShrunkenIndexCheckStep); + assertThat(steps.get(8).getKey(), equalTo(expectedNinthKey)); + assertThat(steps.get(8).getNextStepKey(), equalTo(nextStepKey)); + assertThat(((ShrunkenIndexCheckStep) steps.get(8)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX)); } @Override diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java index 675c24a4195b7..24c1ab1c1cbf1 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java @@ -466,6 +466,24 @@ public void testShrinkAction() throws Exception { expectThrows(ResponseException.class, this::indexDocument); } + public void testShrinkSameShards() throws Exception { + int numberOfShards = randomFrom(1, 2); + String shrunkenIndex = ShrinkAction.SHRUNKEN_INDEX_PREFIX + index; + createIndexWithSettings(index, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numberOfShards) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)); + createNewSingletonPolicy("warm", new ShrinkAction(numberOfShards)); + updatePolicy(index, policy); + assertBusy(() -> { + assertTrue(indexExists(index)); + assertFalse(indexExists(shrunkenIndex)); + assertFalse(aliasExists(shrunkenIndex, index)); + Map settings = getOnlyIndexSettings(index); + assertThat(getStepKeyForIndex(index), equalTo(TerminalPolicyStep.KEY)); + assertThat(settings.get(IndexMetaData.SETTING_NUMBER_OF_SHARDS), equalTo(String.valueOf(numberOfShards))); + assertNull(settings.get(IndexMetaData.INDEX_BLOCKS_WRITE_SETTING.getKey())); + }); + } + public void testShrinkDuringSnapshot() throws Exception { String shrunkenIndex = ShrinkAction.SHRUNKEN_INDEX_PREFIX + index; // Create the repository before taking the snapshot. diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/ExecuteStepsUpdateTask.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/ExecuteStepsUpdateTask.java index 70aa9af2c7277..131330bcb9c99 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/ExecuteStepsUpdateTask.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/ExecuteStepsUpdateTask.java @@ -86,24 +86,30 @@ public ClusterState execute(final ClusterState currentState) throws IOException // either get to a step that isn't a cluster state step or a // cluster state wait step returns not completed while (currentStep instanceof ClusterStateActionStep || currentStep instanceof ClusterStateWaitStep) { - nextStepKey = currentStep.getNextStepKey(); if (currentStep instanceof ClusterStateActionStep) { // cluster state action step so do the action and // move the cluster state to the next step - logger.trace("[{}] performing cluster state action ({}) [{}], next: [{}]", - index.getName(), currentStep.getClass().getSimpleName(), currentStep.getKey(), currentStep.getNextStepKey()); + logger.trace("[{}] performing cluster state action ({}) [{}]", + index.getName(), currentStep.getClass().getSimpleName(), currentStep.getKey()); try { state = ((ClusterStateActionStep) currentStep).performAction(index, state); } catch (Exception exception) { return moveToErrorStep(state, currentStep.getKey(), exception); } - if (currentStep.getNextStepKey() == null) { + // set here to make sure that the clusterProcessed knows to execute the + // correct step if it an async action + nextStepKey = currentStep.getNextStepKey(); + if (nextStepKey == null) { return state; } else { + logger.trace("[{}] moving cluster state to next step [{}]", index.getName(), nextStepKey); state = IndexLifecycleRunner.moveClusterStateToNextStep(index, state, currentStep.getKey(), - currentStep.getNextStepKey(), nowSupplier, false); + nextStepKey, nowSupplier, false); } } else { + // set here to make sure that the clusterProcessed knows to execute the + // correct step if it an async action + nextStepKey = currentStep.getNextStepKey(); // cluster state wait step so evaluate the // condition, if the condition is met move to the // next step, if its not met return the current diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/ExecuteStepsUpdateTaskTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/ExecuteStepsUpdateTaskTests.java index 4611618b2cd24..963ce5d2e2a6f 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/ExecuteStepsUpdateTaskTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/ExecuteStepsUpdateTaskTests.java @@ -268,7 +268,7 @@ public void testClusterActionStepThrowsException() throws IOException { assertThat(currentStepKey, equalTo(new StepKey(firstStepKey.getPhase(), firstStepKey.getAction(), ErrorStep.NAME))); assertThat(firstStep.getExecuteCount(), equalTo(1L)); assertThat(secondStep.getExecuteCount(), equalTo(0L)); - assertThat(task.getNextStepKey(), equalTo(secondStep.getKey())); + assertNull(task.getNextStepKey()); assertThat(lifecycleState.getPhaseTime(), nullValue()); assertThat(lifecycleState.getActionTime(), nullValue()); assertThat(lifecycleState.getStepInfo(), From 6935d3d5fa020055145dfff6b5a7cdeac5ac3d83 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 30 Jan 2019 18:18:30 -0500 Subject: [PATCH 076/100] Temporarily disable BWC for retention lease stats (#38049) This commit temporarily disables BWC testing while backporting a change to expose retention leases in stats. --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 4bd211a12b3b0..164add875eb3b 100644 --- a/build.gradle +++ b/build.gradle @@ -159,8 +159,8 @@ task verifyVersions { * the enabled state of every bwc task. It should be set back to true * after the backport of the backcompat code is complete. */ -final boolean bwc_tests_enabled = true -final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */ +final boolean bwc_tests_enabled = false +final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/37991" /* place a PR link here when committing bwc changes */ if (bwc_tests_enabled == false) { if (bwc_tests_disabled_issue.isEmpty()) { throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false") From dad41c2b7f1c2bb0c2396036d550c5bd5fb381a5 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Wed, 30 Jan 2019 17:38:47 -0600 Subject: [PATCH 077/100] ILM setPriority corrections for a 0 value (#38001) This commit fixes the test case that ensures only a priority less then 0 is used with testNonPositivePriority. This also allows the HLRC to support a value of 0. Closes #37652 --- .../elasticsearch/client/indexlifecycle/SetPriorityAction.java | 2 +- .../client/indexlifecycle/SetPriorityActionTests.java | 2 +- .../xpack/core/indexlifecycle/SetPriorityActionTests.java | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/SetPriorityAction.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/SetPriorityAction.java index 414d2a52ad048..7989c8ee9f145 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/SetPriorityAction.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/SetPriorityAction.java @@ -55,7 +55,7 @@ public static SetPriorityAction parse(XContentParser parser) { } public SetPriorityAction(@Nullable Integer recoveryPriority) { - if (recoveryPriority != null && recoveryPriority <= 0) { + if (recoveryPriority != null && recoveryPriority < 0) { throw new IllegalArgumentException("[" + RECOVERY_PRIORITY_FIELD.getPreferredName() + "] must be 0 or greater"); } this.recoveryPriority = recoveryPriority; diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/SetPriorityActionTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/SetPriorityActionTests.java index f50935a87d398..bec0dd3276400 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/SetPriorityActionTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/SetPriorityActionTests.java @@ -48,7 +48,7 @@ protected boolean supportsUnknownFields() { } public void testNonPositivePriority() { - Exception e = expectThrows(Exception.class, () -> new SetPriorityAction(randomIntBetween(-100, 0))); + Exception e = expectThrows(Exception.class, () -> new SetPriorityAction(randomIntBetween(-100, -1))); assertThat(e.getMessage(), equalTo("[priority] must be 0 or greater")); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/SetPriorityActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/SetPriorityActionTests.java index c0848f5326c40..302cf8cb9c6b8 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/SetPriorityActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/SetPriorityActionTests.java @@ -38,9 +38,8 @@ protected Reader instanceReader() { return SetPriorityAction::new; } - @AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/37652") public void testNonPositivePriority() { - Exception e = expectThrows(Exception.class, () -> new SetPriorityAction(randomIntBetween(-100, 0))); + Exception e = expectThrows(Exception.class, () -> new SetPriorityAction(randomIntBetween(-100, -1))); assertThat(e.getMessage(), equalTo("[priority] must be 0 or greater")); } From b11732104fa5a611b5d47bc59efd594e9cbdab89 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Wed, 30 Jan 2019 20:14:59 -0500 Subject: [PATCH 078/100] Move watcher to use seq# and primary term for concurrency control (#37977) * move watcher to seq# occ * top level set * fix parsing and missing setters * share toXContent for PutResponse and rest end point * fix redacted password * fix username reference * fix deactivate-watch.asciidoc have seq no references * add seq# + term to activate-watch.asciidoc * more doc fixes --- .../client/RequestConverters.java | 15 ++ .../client/WatcherRequestConverters.java | 5 +- .../client/watcher/GetWatchResponse.java | 29 ++- .../client/watcher/PutWatchRequest.java | 57 ++++++ .../client/watcher/PutWatchResponse.java | 31 +++- .../client/watcher/PutWatchResponseTests.java | 6 +- .../en/rest-api/watcher/ack-watch.asciidoc | 6 + .../rest-api/watcher/activate-watch.asciidoc | 2 + .../watcher/deactivate-watch.asciidoc | 2 + .../en/rest-api/watcher/get-watch.asciidoc | 2 + .../xpack/watcher/PutWatchRequest.java | 89 ++++++++- .../xpack/watcher/PutWatchResponse.java | 45 ++++- .../actions/get/GetWatchResponse.java | 34 +++- .../xpack/core/watcher/watch/Watch.java | 25 ++- .../xpack/core/watcher/watch/WatchField.java | 1 - .../xpack/watcher/GetWatchResponseTests.java | 9 +- .../xpack/watcher/PutWatchResponseTests.java | 7 +- .../api/xpack.watcher.put_watch.json | 8 + .../80_put_get_watch_with_passwords.yml | 169 ++++++++++++++++++ .../watcher/WatcherIndexingListener.java | 3 +- .../xpack/watcher/WatcherService.java | 5 +- .../watcher/execution/ExecutionService.java | 6 +- .../rest/action/RestExecuteWatchAction.java | 3 +- .../rest/action/RestGetWatchAction.java | 16 +- .../rest/action/RestPutWatchAction.java | 8 +- .../actions/ack/TransportAckWatchAction.java | 16 +- .../TransportActivateWatchAction.java | 3 +- .../execute/TransportExecuteWatchAction.java | 8 +- .../actions/get/TransportGetWatchAction.java | 8 +- .../actions/put/TransportPutWatchAction.java | 19 +- .../xpack/watcher/watch/WatchParser.java | 35 ++-- .../watcher/WatcherIndexingListenerTests.java | 9 +- .../xpack/watcher/WatcherServiceTests.java | 3 +- .../execution/ExecutionServiceTests.java | 17 +- .../xpack/watcher/test/WatcherTestUtils.java | 5 +- .../bench/ScheduleEngineTriggerBenchmark.java | 2 +- .../test/integration/WatchAckTests.java | 9 +- .../ack/TransportAckWatchActionTests.java | 3 +- .../put/TransportPutWatchActionTests.java | 4 +- .../engine/TickerScheduleEngineTests.java | 4 +- .../xpack/watcher/watch/WatchTests.java | 23 ++- 41 files changed, 624 insertions(+), 127 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index 7a6a562e4fb7f..1286e083d8203 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -76,6 +76,7 @@ import org.elasticsearch.index.reindex.DeleteByQueryRequest; import org.elasticsearch.index.reindex.ReindexRequest; import org.elasticsearch.index.reindex.UpdateByQueryRequest; +import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.script.mustache.MultiSearchTemplateRequest; import org.elasticsearch.script.mustache.SearchTemplateRequest; @@ -885,6 +886,20 @@ Params withVersionType(VersionType versionType) { return this; } + Params withIfSeqNo(long ifSeqNo) { + if (ifSeqNo != SequenceNumbers.UNASSIGNED_SEQ_NO) { + return putParam("if_seq_no", Long.toString(ifSeqNo)); + } + return this; + } + + Params withIfPrimaryTerm(long ifPrimaryTerm) { + if (ifPrimaryTerm != SequenceNumbers.UNASSIGNED_PRIMARY_TERM) { + return putParam("if_primary_term", Long.toString(ifPrimaryTerm)); + } + return this; + } + Params withWaitForActiveShards(ActiveShardCount activeShardCount) { return withWaitForActiveShards(activeShardCount, ActiveShardCount.DEFAULT); } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/WatcherRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/WatcherRequestConverters.java index a5c6d0dd1810e..34fb826d62382 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/WatcherRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/WatcherRequestConverters.java @@ -69,7 +69,10 @@ static Request putWatch(PutWatchRequest putWatchRequest) { .build(); Request request = new Request(HttpPut.METHOD_NAME, endpoint); - RequestConverters.Params params = new RequestConverters.Params(request).withVersion(putWatchRequest.getVersion()); + RequestConverters.Params params = new RequestConverters.Params(request) + .withVersion(putWatchRequest.getVersion()) + .withIfSeqNo(putWatchRequest.ifSeqNo()) + .withIfPrimaryTerm(putWatchRequest.ifPrimaryTerm()); if (putWatchRequest.isActive() == false) { params.putParam("active", "false"); } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/watcher/GetWatchResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/watcher/GetWatchResponse.java index 9f5934b33eb30..83727003106e9 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/watcher/GetWatchResponse.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/watcher/GetWatchResponse.java @@ -31,9 +31,14 @@ import java.util.Map; import java.util.Objects; +import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM; +import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; + public class GetWatchResponse { private final String id; private final long version; + private final long seqNo; + private final long primaryTerm; private final WatchStatus status; private final BytesReference source; @@ -43,15 +48,18 @@ public class GetWatchResponse { * Ctor for missing watch */ public GetWatchResponse(String id) { - this(id, Versions.NOT_FOUND, null, null, null); + this(id, Versions.NOT_FOUND, UNASSIGNED_SEQ_NO, UNASSIGNED_PRIMARY_TERM, null, null, null); } - public GetWatchResponse(String id, long version, WatchStatus status, BytesReference source, XContentType xContentType) { + public GetWatchResponse(String id, long version, long seqNo, long primaryTerm, WatchStatus status, + BytesReference source, XContentType xContentType) { this.id = id; this.version = version; this.status = status; this.source = source; this.xContentType = xContentType; + this.seqNo = seqNo; + this.primaryTerm = primaryTerm; } public String getId() { @@ -62,6 +70,14 @@ public long getVersion() { return version; } + public long getSeqNo() { + return seqNo; + } + + public long getPrimaryTerm() { + return primaryTerm; + } + public boolean isFound() { return version != Versions.NOT_FOUND; } @@ -111,6 +127,8 @@ public int hashCode() { private static final ParseField ID_FIELD = new ParseField("_id"); private static final ParseField FOUND_FIELD = new ParseField("found"); private static final ParseField VERSION_FIELD = new ParseField("_version"); + private static final ParseField SEQ_NO_FIELD = new ParseField("_seq_no"); + private static final ParseField PRIMARY_TERM_FIELD = new ParseField("_primary_term"); private static final ParseField STATUS_FIELD = new ParseField("status"); private static final ParseField WATCH_FIELD = new ParseField("watch"); @@ -119,9 +137,10 @@ public int hashCode() { a -> { boolean isFound = (boolean) a[1]; if (isFound) { - XContentBuilder builder = (XContentBuilder) a[4]; + XContentBuilder builder = (XContentBuilder) a[6]; BytesReference source = BytesReference.bytes(builder); - return new GetWatchResponse((String) a[0], (long) a[2], (WatchStatus) a[3], source, builder.contentType()); + return new GetWatchResponse((String) a[0], (long) a[2], (long) a[3], (long) a[4], (WatchStatus) a[5], + source, builder.contentType()); } else { return new GetWatchResponse((String) a[0]); } @@ -131,6 +150,8 @@ public int hashCode() { PARSER.declareString(ConstructingObjectParser.constructorArg(), ID_FIELD); PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), FOUND_FIELD); PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), VERSION_FIELD); + PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), SEQ_NO_FIELD); + PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), PRIMARY_TERM_FIELD); PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (parser, context) -> WatchStatus.parse(parser), STATUS_FIELD); PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/watcher/PutWatchRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/watcher/PutWatchRequest.java index 88f47aeaeee7a..8b83970723dd2 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/watcher/PutWatchRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/watcher/PutWatchRequest.java @@ -23,10 +23,14 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.seqno.SequenceNumbers; import java.util.Objects; import java.util.regex.Pattern; +import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM; +import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; + /** * This request class contains the data needed to create a watch along with the name of the watch. * The name of the watch will become the ID of the indexed document. @@ -40,6 +44,9 @@ public final class PutWatchRequest implements Validatable { private final XContentType xContentType; private boolean active = true; private long version = Versions.MATCH_ANY; + private long ifSeqNo = SequenceNumbers.UNASSIGNED_SEQ_NO; + private long ifPrimaryTerm = UNASSIGNED_PRIMARY_TERM; + public PutWatchRequest(String id, BytesReference source, XContentType xContentType) { Objects.requireNonNull(id, "watch id is missing"); @@ -96,6 +103,56 @@ public void setVersion(long version) { this.version = version; } + /** + * only performs this put request if the watch's last modification was assigned the given + * sequence number. Must be used in combination with {@link #setIfPrimaryTerm(long)} + * + * If the watch's last modification was assigned a different sequence number a + * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown. + */ + public PutWatchRequest setIfSeqNo(long seqNo) { + if (seqNo < 0 && seqNo != UNASSIGNED_SEQ_NO) { + throw new IllegalArgumentException("sequence numbers must be non negative. got [" + seqNo + "]."); + } + ifSeqNo = seqNo; + return this; + } + + /** + * only performs this put request if the watch's last modification was assigned the given + * primary term. Must be used in combination with {@link #setIfSeqNo(long)} + * + * If the watch last modification was assigned a different term a + * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown. + */ + public PutWatchRequest setIfPrimaryTerm(long term) { + if (term < 0) { + throw new IllegalArgumentException("primary term must be non negative. got [" + term + "]"); + } + ifPrimaryTerm = term; + return this; + } + + /** + * If set, only perform this put watch request if the watch's last modification was assigned this sequence number. + * If the watch last last modification was assigned a different sequence number a + * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown. + */ + public long ifSeqNo() { + return ifSeqNo; + } + + /** + * If set, only perform this put watch request if the watch's last modification was assigned this primary term. + * + * If the watch's last modification was assigned a different term a + * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown. + */ + public long ifPrimaryTerm() { + return ifPrimaryTerm; + } + + public static boolean isValidId(String id) { return Strings.isEmpty(id) == false && NO_WS_PATTERN.matcher(id).matches(); } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/watcher/PutWatchResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/watcher/PutWatchResponse.java index 8f7070b2565a2..61742b84e1219 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/watcher/PutWatchResponse.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/watcher/PutWatchResponse.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.ParseField; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.seqno.SequenceNumbers; import java.io.IOException; import java.util.Objects; @@ -32,20 +33,26 @@ public class PutWatchResponse { static { PARSER.declareString(PutWatchResponse::setId, new ParseField("_id")); + PARSER.declareLong(PutWatchResponse::setSeqNo, new ParseField("_seq_no")); + PARSER.declareLong(PutWatchResponse::setPrimaryTerm, new ParseField("_primary_term")); PARSER.declareLong(PutWatchResponse::setVersion, new ParseField("_version")); PARSER.declareBoolean(PutWatchResponse::setCreated, new ParseField("created")); } private String id; private long version; + private long seqNo = SequenceNumbers.UNASSIGNED_SEQ_NO; + private long primaryTerm = SequenceNumbers.UNASSIGNED_PRIMARY_TERM; private boolean created; public PutWatchResponse() { } - public PutWatchResponse(String id, long version, boolean created) { + public PutWatchResponse(String id, long version, long seqNo, long primaryTerm, boolean created) { this.id = id; this.version = version; + this.seqNo = seqNo; + this.primaryTerm = primaryTerm; this.created = created; } @@ -57,6 +64,14 @@ private void setVersion(long version) { this.version = version; } + private void setSeqNo(long seqNo) { + this.seqNo = seqNo; + } + + private void setPrimaryTerm(long primaryTerm) { + this.primaryTerm = primaryTerm; + } + private void setCreated(boolean created) { this.created = created; } @@ -69,6 +84,14 @@ public long getVersion() { return version; } + public long getSeqNo() { + return seqNo; + } + + public long getPrimaryTerm() { + return primaryTerm; + } + public boolean isCreated() { return created; } @@ -80,12 +103,14 @@ public boolean equals(Object o) { PutWatchResponse that = (PutWatchResponse) o; - return Objects.equals(id, that.id) && Objects.equals(version, that.version) && Objects.equals(created, that.created); + return Objects.equals(id, that.id) && Objects.equals(version, that.version) + && Objects.equals(seqNo, that.seqNo) + && Objects.equals(primaryTerm, that.primaryTerm) && Objects.equals(created, that.created); } @Override public int hashCode() { - return Objects.hash(id, version, created); + return Objects.hash(id, version, seqNo, primaryTerm, created); } public static PutWatchResponse fromXContent(XContentParser parser) throws IOException { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/PutWatchResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/PutWatchResponseTests.java index af327abc1a728..d358f5c8955ae 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/PutWatchResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/PutWatchResponseTests.java @@ -41,14 +41,18 @@ private static XContentBuilder toXContent(PutWatchResponse response, XContentBui return builder.startObject() .field("_id", response.getId()) .field("_version", response.getVersion()) + .field("_seq_no", response.getSeqNo()) + .field("_primary_term", response.getPrimaryTerm()) .field("created", response.isCreated()) .endObject(); } private static PutWatchResponse createTestInstance() { String id = randomAlphaOfLength(10); + long seqNo = randomNonNegativeLong(); + long primaryTerm = randomLongBetween(1, 200); long version = randomLongBetween(1, 10); boolean created = randomBoolean(); - return new PutWatchResponse(id, version, created); + return new PutWatchResponse(id, version, seqNo, primaryTerm, created); } } diff --git a/x-pack/docs/en/rest-api/watcher/ack-watch.asciidoc b/x-pack/docs/en/rest-api/watcher/ack-watch.asciidoc index f1e281819eec3..9e97e8d754a0e 100644 --- a/x-pack/docs/en/rest-api/watcher/ack-watch.asciidoc +++ b/x-pack/docs/en/rest-api/watcher/ack-watch.asciidoc @@ -92,6 +92,8 @@ The action state of a newly-created watch is `awaits_successful_execution`: -------------------------------------------------- { "found": true, + "_seq_no": 0, + "_primary_term": 1, "_version": 1, "_id": "my_watch", "status": { @@ -136,6 +138,8 @@ and the action is now in `ackable` state: { "found": true, "_id": "my_watch", + "_seq_no": 1, + "_primary_term": 1, "_version": 2, "status": { "version": 2, @@ -185,6 +189,8 @@ GET _watcher/watch/my_watch { "found": true, "_id": "my_watch", + "_seq_no": 2, + "_primary_term": 1, "_version": 3, "status": { "version": 3, diff --git a/x-pack/docs/en/rest-api/watcher/activate-watch.asciidoc b/x-pack/docs/en/rest-api/watcher/activate-watch.asciidoc index ae49ba1f369c8..74a98a00fa423 100644 --- a/x-pack/docs/en/rest-api/watcher/activate-watch.asciidoc +++ b/x-pack/docs/en/rest-api/watcher/activate-watch.asciidoc @@ -44,6 +44,8 @@ GET _watcher/watch/my_watch { "found": true, "_id": "my_watch", + "_seq_no": 0, + "_primary_term": 1, "_version": 1, "status": { "state" : { diff --git a/x-pack/docs/en/rest-api/watcher/deactivate-watch.asciidoc b/x-pack/docs/en/rest-api/watcher/deactivate-watch.asciidoc index 96fad5a702854..59625c1391119 100644 --- a/x-pack/docs/en/rest-api/watcher/deactivate-watch.asciidoc +++ b/x-pack/docs/en/rest-api/watcher/deactivate-watch.asciidoc @@ -44,6 +44,8 @@ GET _watcher/watch/my_watch "found": true, "_id": "my_watch", "_version": 1, + "_seq_no": 0, + "_primary_term": 1, "status": { "state" : { "active" : true, diff --git a/x-pack/docs/en/rest-api/watcher/get-watch.asciidoc b/x-pack/docs/en/rest-api/watcher/get-watch.asciidoc index 87a13d0829f9b..bd26c6f26748c 100644 --- a/x-pack/docs/en/rest-api/watcher/get-watch.asciidoc +++ b/x-pack/docs/en/rest-api/watcher/get-watch.asciidoc @@ -44,6 +44,8 @@ Response: { "found": true, "_id": "my_watch", + "_seq_no": 0, + "_primary_term": 1, "_version": 1, "status": { <1> "version": 1, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchRequest.java index 7997d853db37a..fbee2963ad92d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchRequest.java @@ -5,19 +5,24 @@ */ package org.elasticsearch.protocol.xpack.watcher; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.ValidateActions; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.seqno.SequenceNumbers; import java.io.IOException; import java.util.regex.Pattern; +import static org.elasticsearch.action.ValidateActions.addValidationError; +import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM; +import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; + /** * This request class contains the data needed to create a watch along with the name of the watch. * The name of the watch will become the ID of the indexed document. @@ -32,6 +37,9 @@ public final class PutWatchRequest extends ActionRequest { private boolean active = true; private long version = Versions.MATCH_ANY; + private long ifSeqNo = SequenceNumbers.UNASSIGNED_SEQ_NO; + private long ifPrimaryTerm = UNASSIGNED_PRIMARY_TERM; + public PutWatchRequest() {} public PutWatchRequest(StreamInput in) throws IOException { @@ -52,6 +60,13 @@ public void readFrom(StreamInput in) throws IOException { active = in.readBoolean(); xContentType = in.readEnum(XContentType.class); version = in.readZLong(); + if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + ifSeqNo = in.readZLong(); + ifPrimaryTerm = in.readVLong(); + } else { + ifSeqNo = SequenceNumbers.UNASSIGNED_SEQ_NO; + ifPrimaryTerm = UNASSIGNED_PRIMARY_TERM; + } } @Override @@ -62,6 +77,10 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(active); out.writeEnum(xContentType); out.writeZLong(version); + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + out.writeZLong(ifSeqNo); + out.writeVLong(ifPrimaryTerm); + } } /** @@ -122,20 +141,80 @@ public void setVersion(long version) { this.version = version; } + /** + * only performs this put request if the watch's last modification was assigned the given + * sequence number. Must be used in combination with {@link #setIfPrimaryTerm(long)} + * + * If the watch's last modification was assigned a different sequence number a + * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown. + */ + public PutWatchRequest setIfSeqNo(long seqNo) { + if (seqNo < 0 && seqNo != UNASSIGNED_SEQ_NO) { + throw new IllegalArgumentException("sequence numbers must be non negative. got [" + seqNo + "]."); + } + ifSeqNo = seqNo; + return this; + } + + /** + * only performs this put request if the watch's last modification was assigned the given + * primary term. Must be used in combination with {@link #setIfSeqNo(long)} + * + * If the watch last modification was assigned a different term a + * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown. + */ + public PutWatchRequest setIfPrimaryTerm(long term) { + if (term < 0) { + throw new IllegalArgumentException("primary term must be non negative. got [" + term + "]"); + } + ifPrimaryTerm = term; + return this; + } + + /** + * If set, only perform this put watch request if the watch's last modification was assigned this sequence number. + * If the watch last last modification was assigned a different sequence number a + * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown. + */ + public long getIfSeqNo() { + return ifSeqNo; + } + + /** + * If set, only perform this put watch request if the watch's last modification was assigned this primary term. + * + * If the watch's last modification was assigned a different term a + * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown. + */ + public long getIfPrimaryTerm() { + return ifPrimaryTerm; + } + @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; if (id == null) { - validationException = ValidateActions.addValidationError("watch id is missing", validationException); + validationException = addValidationError("watch id is missing", validationException); } else if (isValidId(id) == false) { - validationException = ValidateActions.addValidationError("watch id contains whitespace", validationException); + validationException = addValidationError("watch id contains whitespace", validationException); } if (source == null) { - validationException = ValidateActions.addValidationError("watch source is missing", validationException); + validationException = addValidationError("watch source is missing", validationException); } if (xContentType == null) { - validationException = ValidateActions.addValidationError("request body is missing", validationException); + validationException = addValidationError("request body is missing", validationException); + } + if (ifSeqNo != UNASSIGNED_SEQ_NO && version != Versions.MATCH_ANY) { + validationException = addValidationError("compare and write operations can not use versioning", validationException); } + if (ifPrimaryTerm == UNASSIGNED_PRIMARY_TERM && ifSeqNo != UNASSIGNED_SEQ_NO) { + validationException = addValidationError("ifSeqNo is set, but primary term is [0]", validationException); + } + if (ifPrimaryTerm != UNASSIGNED_PRIMARY_TERM && ifSeqNo == UNASSIGNED_SEQ_NO) { + validationException = + addValidationError("ifSeqNo is unassigned, but primary term is [" + ifPrimaryTerm + "]", validationException); + } + return validationException; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchResponse.java index f6e55ff555339..ab8496f839a3c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchResponse.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.protocol.xpack.watcher; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; @@ -13,6 +14,7 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.seqno.SequenceNumbers; import java.io.IOException; import java.util.Objects; @@ -24,19 +26,25 @@ public class PutWatchResponse extends ActionResponse implements ToXContentObject static { PARSER.declareString(PutWatchResponse::setId, new ParseField("_id")); PARSER.declareLong(PutWatchResponse::setVersion, new ParseField("_version")); + PARSER.declareLong(PutWatchResponse::setSeqNo, new ParseField("_seq_no")); + PARSER.declareLong(PutWatchResponse::setPrimaryTerm, new ParseField("_primary_term")); PARSER.declareBoolean(PutWatchResponse::setCreated, new ParseField("created")); } private String id; private long version; + private long seqNo = SequenceNumbers.UNASSIGNED_SEQ_NO; + private long primaryTerm = SequenceNumbers.UNASSIGNED_PRIMARY_TERM; private boolean created; public PutWatchResponse() { } - public PutWatchResponse(String id, long version, boolean created) { + public PutWatchResponse(String id, long version, long seqNo, long primaryTerm, boolean created) { this.id = id; this.version = version; + this.seqNo = seqNo; + this.primaryTerm = primaryTerm; this.created = created; } @@ -48,6 +56,14 @@ private void setVersion(long version) { this.version = version; } + private void setSeqNo(long seqNo) { + this.seqNo = seqNo; + } + + private void setPrimaryTerm(long primaryTerm) { + this.primaryTerm = primaryTerm; + } + private void setCreated(boolean created) { this.created = created; } @@ -60,6 +76,14 @@ public long getVersion() { return version; } + public long getSeqNo() { + return seqNo; + } + + public long getPrimaryTerm() { + return primaryTerm; + } + public boolean isCreated() { return created; } @@ -71,12 +95,14 @@ public boolean equals(Object o) { PutWatchResponse that = (PutWatchResponse) o; - return Objects.equals(id, that.id) && Objects.equals(version, that.version) && Objects.equals(created, that.created); + return Objects.equals(id, that.id) && Objects.equals(version, that.version) + && Objects.equals(seqNo, that.seqNo) + && Objects.equals(primaryTerm, that.primaryTerm) && Objects.equals(created, that.created); } @Override public int hashCode() { - return Objects.hash(id, version, created); + return Objects.hash(id, version, seqNo, primaryTerm, created); } @Override @@ -84,6 +110,10 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(id); out.writeVLong(version); + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + out.writeZLong(seqNo); + out.writeVLong(primaryTerm); + } out.writeBoolean(created); } @@ -92,6 +122,13 @@ public void readFrom(StreamInput in) throws IOException { super.readFrom(in); id = in.readString(); version = in.readVLong(); + if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + seqNo = in.readZLong(); + primaryTerm = in.readVLong(); + } else { + seqNo = SequenceNumbers.UNASSIGNED_SEQ_NO; + primaryTerm = SequenceNumbers.UNASSIGNED_PRIMARY_TERM; + } created = in.readBoolean(); } @@ -100,6 +137,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder.startObject() .field("_id", id) .field("_version", version) + .field("_seq_no", seqNo) + .field("_primary_term", primaryTerm) .field("created", created) .endObject(); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/get/GetWatchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/get/GetWatchResponse.java index d92ae1dcc4626..e61632f7d1d4c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/get/GetWatchResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/get/GetWatchResponse.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.core.watcher.transport.actions.get; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; @@ -12,6 +13,7 @@ import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource; import org.elasticsearch.xpack.core.watcher.watch.WatchStatus; @@ -25,6 +27,8 @@ public class GetWatchResponse extends ActionResponse implements ToXContent { private boolean found; private XContentSource source; private long version; + private long seqNo; + private long primaryTerm; public GetWatchResponse() { } @@ -38,17 +42,21 @@ public GetWatchResponse(String id) { this.found = false; this.source = null; this.version = Versions.NOT_FOUND; + this.seqNo = SequenceNumbers.UNASSIGNED_SEQ_NO; + this.primaryTerm = SequenceNumbers.UNASSIGNED_PRIMARY_TERM; } /** * ctor for found watch */ - public GetWatchResponse(String id, long version, WatchStatus status, XContentSource source) { + public GetWatchResponse(String id, long version, long seqNo, long primaryTerm, WatchStatus status, XContentSource source) { this.id = id; this.status = status; this.found = true; this.source = source; this.version = version; + this.seqNo = seqNo; + this.primaryTerm = primaryTerm; } public String getId() { @@ -71,6 +79,14 @@ public long getVersion() { return version; } + public long getSeqNo() { + return seqNo; + } + + public long getPrimaryTerm() { + return primaryTerm; + } + @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); @@ -80,10 +96,16 @@ public void readFrom(StreamInput in) throws IOException { status = WatchStatus.read(in); source = XContentSource.readFrom(in); version = in.readZLong(); + if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + seqNo = in.readZLong(); + primaryTerm = in.readVLong(); + } } else { status = null; source = null; version = Versions.NOT_FOUND; + seqNo = SequenceNumbers.UNASSIGNED_SEQ_NO; + primaryTerm = SequenceNumbers.UNASSIGNED_PRIMARY_TERM; } } @@ -96,6 +118,10 @@ public void writeTo(StreamOutput out) throws IOException { status.writeTo(out); XContentSource.writeTo(source, out); out.writeZLong(version); + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + out.writeZLong(seqNo); + out.writeVLong(primaryTerm); + } } } @@ -105,6 +131,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("_id", id); if (found) { builder.field("_version", version); + builder.field("_seq_no", seqNo); + builder.field("_primary_term", primaryTerm); builder.field("status", status, params); builder.field("watch", source, params); } @@ -116,7 +144,7 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; GetWatchResponse that = (GetWatchResponse) o; - return version == that.version && + return version == that.version && seqNo == that.seqNo && primaryTerm == that.primaryTerm && Objects.equals(id, that.id) && Objects.equals(status, that.status) && Objects.equals(source, that.source); @@ -124,7 +152,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(id, status, version); + return Objects.hash(id, status, version, seqNo, primaryTerm); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/watch/Watch.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/watch/Watch.java index 75034752c3cc6..e34b0a15e7133 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/watch/Watch.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/watch/Watch.java @@ -9,6 +9,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.xpack.core.watcher.actions.ActionStatus; import org.elasticsearch.xpack.core.watcher.actions.ActionWrapper; import org.elasticsearch.xpack.core.watcher.condition.ExecutableCondition; @@ -37,11 +38,12 @@ public class Watch implements ToXContentObject { @Nullable private final Map metadata; private final WatchStatus status; - private transient long version; + private final long sourceSeqNo; + private final long sourcePrimaryTerm; public Watch(String id, Trigger trigger, ExecutableInput input, ExecutableCondition condition, @Nullable ExecutableTransform transform, @Nullable TimeValue throttlePeriod, List actions, @Nullable Map metadata, - WatchStatus status, long version) { + WatchStatus status, long sourceSeqNo, long sourcePrimaryTerm) { this.id = id; this.trigger = trigger; this.input = input; @@ -51,7 +53,8 @@ public Watch(String id, Trigger trigger, ExecutableInput input, ExecutableCondit this.throttlePeriod = throttlePeriod; this.metadata = metadata; this.status = status; - this.version = version; + this.sourceSeqNo = sourceSeqNo; + this.sourcePrimaryTerm = sourcePrimaryTerm; } public String id() { @@ -88,12 +91,20 @@ public WatchStatus status() { return status; } - public long version() { - return version; + /** + * The sequence number of the document that was used to create this watch, {@link SequenceNumbers#UNASSIGNED_SEQ_NO} + * if the watch wasn't read from a document + ***/ + public long getSourceSeqNo() { + return sourceSeqNo; } - public void version(long version) { - this.version = version; + /** + * The primary term of the document that was used to create this watch, {@link SequenceNumbers#UNASSIGNED_PRIMARY_TERM} + * if the watch wasn't read from a document + ***/ + public long getSourcePrimaryTerm() { + return sourcePrimaryTerm; } /** diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/watch/WatchField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/watch/WatchField.java index dbc3ce76c9517..6f6a1955927d9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/watch/WatchField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/watch/WatchField.java @@ -17,7 +17,6 @@ public final class WatchField { public static final ParseField THROTTLE_PERIOD_HUMAN = new ParseField("throttle_period"); public static final ParseField METADATA = new ParseField("metadata"); public static final ParseField STATUS = new ParseField("status"); - public static final ParseField VERSION = new ParseField("_version"); public static final String ALL_ACTIONS_ID = "_all"; private WatchField() {} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/watcher/GetWatchResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/watcher/GetWatchResponseTests.java index 1c00b6fd9dc27..13f4b0fb5d3b4 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/watcher/GetWatchResponseTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/watcher/GetWatchResponseTests.java @@ -74,6 +74,7 @@ protected void assertEqualInstances(GetWatchResponse expectedInstance, GetWatchR throw new AssertionError(e); } newInstance = new GetWatchResponse(newInstance.getId(), newInstance.getVersion(), + newInstance.getSeqNo(), newInstance.getPrimaryTerm(), newInstance.getStatus(), new XContentSource(newSource, expectedInstance.getSource().getContentType())); } super.assertEqualInstances(expectedInstance, newInstance); @@ -91,9 +92,11 @@ protected GetWatchResponse createTestInstance() { return new GetWatchResponse(id); } long version = randomLongBetween(0, 10); + long seqNo = randomNonNegativeLong(); + long primaryTerm = randomLongBetween(1, 2000); WatchStatus status = randomWatchStatus(); BytesReference source = simpleWatch(); - return new GetWatchResponse(id, version, status, new XContentSource(source, XContentType.JSON)); + return new GetWatchResponse(id, version, seqNo, primaryTerm, status, new XContentSource(source, XContentType.JSON)); } private static BytesReference simpleWatch() { @@ -170,8 +173,8 @@ public org.elasticsearch.client.watcher.GetWatchResponse doHlrcParseInstance(XCo @Override public GetWatchResponse convertHlrcToInternal(org.elasticsearch.client.watcher.GetWatchResponse instance) { if (instance.isFound()) { - return new GetWatchResponse(instance.getId(), instance.getVersion(), convertHlrcToInternal(instance.getStatus()), - new XContentSource(instance.getSource(), instance.getContentType())); + return new GetWatchResponse(instance.getId(), instance.getVersion(), instance.getSeqNo(), instance.getPrimaryTerm(), + convertHlrcToInternal(instance.getStatus()), new XContentSource(instance.getSource(), instance.getContentType())); } else { return new GetWatchResponse(instance.getId()); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/watcher/PutWatchResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/watcher/PutWatchResponseTests.java index 8ea4a84daed95..975d842b79537 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/watcher/PutWatchResponseTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/watcher/PutWatchResponseTests.java @@ -16,9 +16,11 @@ public class PutWatchResponseTests extends @Override protected PutWatchResponse createTestInstance() { String id = randomAlphaOfLength(10); + long seqNo = randomNonNegativeLong(); + long primaryTerm = randomLongBetween(1, 20); long version = randomLongBetween(1, 10); boolean created = randomBoolean(); - return new PutWatchResponse(id, version, created); + return new PutWatchResponse(id, version, seqNo, primaryTerm, created); } @Override @@ -33,7 +35,8 @@ public org.elasticsearch.client.watcher.PutWatchResponse doHlrcParseInstance(XCo @Override public PutWatchResponse convertHlrcToInternal(org.elasticsearch.client.watcher.PutWatchResponse instance) { - return new PutWatchResponse(instance.getId(), instance.getVersion(), instance.isCreated()); + return new PutWatchResponse(instance.getId(), instance.getVersion(), instance.getSeqNo(), instance.getPrimaryTerm(), + instance.isCreated()); } @Override diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.watcher.put_watch.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.watcher.put_watch.json index aabbc8aef7f4c..438f2e4ee7637 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.watcher.put_watch.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.watcher.put_watch.json @@ -20,6 +20,14 @@ "version" : { "type" : "number", "description" : "Explicit version number for concurrency control" + }, + "if_seq_no" : { + "type" : "number", + "description" : "only update the watch if the last operation that has changed the watch has the specified sequence number" + }, + "if_primary_term" : { + "type" : "number", + "description" : "only update the watch if the last operation that has changed the watch has the specified primary term" } } }, diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/watcher/put_watch/80_put_get_watch_with_passwords.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/watcher/put_watch/80_put_get_watch_with_passwords.yml index 03bc40c9372ba..f61bc91f2ce35 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/watcher/put_watch/80_put_get_watch_with_passwords.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/watcher/put_watch/80_put_get_watch_with_passwords.yml @@ -221,6 +221,175 @@ setup: } } +--- +"Test putting a watch with a redacted password with old seq no returns an error": + - skip: + version: " - 6.99.99" + reason: seq no powered concurrency was added in 7.0.0 + + # version 1 + - do: + xpack.watcher.put_watch: + id: "watch_with_seq_no" + body: > + { + "trigger": { + "schedule" : { "cron" : "0 0 0 1 * ? 2099" } + }, + "input": { + "http" : { + "request" : { + "host" : "host.domain", + "port" : 9200, + "path" : "/myservice", + "auth" : { + "basic" : { + "username" : "user", + "password" : "pass" + } + } + } + } + }, + "actions": { + "logging": { + "logging": { + "text": "Log me Amadeus!" + } + } + } + } + + - set: { "_seq_no": seqNo } + - set: { "_primary_term" : primaryTerm } + + # using optimistic concurrency control, this one will loose + # as if two users in the watch UI tried to update the same watch + - do: + catch: conflict + xpack.watcher.put_watch: + id: "watch_with_seq_no" + if_seq_no: 123034 + if_primary_term: $primaryTerm + body: > + { + "trigger": { + "schedule" : { "cron" : "0 0 0 1 * ? 2099" } + }, + "input": { + "http" : { + "request" : { + "host" : "host.domain", + "port" : 9200, + "path" : "/myservice", + "auth" : { + "basic" : { + "username" : "user", + "password" : "::es_redacted::" + } + } + } + } + }, + "actions": { + "logging": { + "logging": { + "text": "Log me Amadeus!" + } + } + } + } + + - do: + catch: conflict + xpack.watcher.put_watch: + id: "watch_with_seq_no" + if_seq_no: $seqNo + if_primary_term: 234242423 + body: > + { + "trigger": { + "schedule" : { "cron" : "0 0 0 1 * ? 2099" } + }, + "input": { + "http" : { + "request" : { + "host" : "host.domain", + "port" : 9200, + "path" : "/myservice", + "auth" : { + "basic" : { + "username" : "user", + "password" : "::es_redacted::" + } + } + } + } + }, + "actions": { + "logging": { + "logging": { + "text": "Log me Amadeus!" + } + } + } + } + + - do: + xpack.watcher.put_watch: + id: "watch_with_seq_no" + if_seq_no: $seqNo + if_primary_term: $primaryTerm + body: > + { + "trigger": { + "schedule" : { "cron" : "0 0 0 1 * ? 2099" } + }, + "input": { + "http" : { + "request" : { + "host" : "host.domain", + "port" : 9200, + "path" : "/myservice", + "auth" : { + "basic" : { + "username" : "new_user", + "password" : "::es_redacted::" + } + } + } + } + }, + "actions": { + "logging": { + "logging": { + "text": "Log me Amadeus!" + } + } + } + } + + - do: + search: + rest_total_hits_as_int: true + index: .watches + body: > + { + "query": { + "term": { + "_id": { + "value": "watch_with_seq_no" + } + } + } + } + + + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "watch_with_seq_no" } + - match: { hits.hits.0._source.input.http.request.auth.basic.username: "new_user" } + - match: { hits.hits.0._source.input.http.request.auth.basic.password: "pass" } + --- "Test putting a watch with a redacted password with current version works": diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherIndexingListener.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherIndexingListener.java index 1974dd055ddc8..1f2ddb4ed07a6 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherIndexingListener.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherIndexingListener.java @@ -103,7 +103,8 @@ public Engine.Index preIndex(ShardId shardId, Engine.Index operation) { if (isWatchDocument(shardId.getIndexName(), operation.type())) { DateTime now = new DateTime(clock.millis(), UTC); try { - Watch watch = parser.parseWithSecrets(operation.id(), true, operation.source(), now, XContentType.JSON); + Watch watch = parser.parseWithSecrets(operation.id(), true, operation.source(), now, XContentType.JSON, + operation.getIfSeqNo(), operation.getIfPrimaryTerm()); ShardAllocationConfiguration shardAllocationConfiguration = configuration.localShards.get(shardId); if (shardAllocationConfiguration == null) { logger.debug("no distributed watch execution info found for watch [{}] on shard [{}], got configuration for {}", diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherService.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherService.java index 796d06b0b81ea..c96203bd64222 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherService.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherService.java @@ -298,7 +298,7 @@ private Collection loadWatches(ClusterState clusterState) { .source(new SearchSourceBuilder() .size(scrollSize) .sort(SortBuilders.fieldSort("_doc")) - .version(true)); + .seqNoAndPrimaryTerm(true)); response = client.search(searchRequest).actionGet(defaultSearchTimeout); if (response.getTotalShards() != response.getSuccessfulShards()) { @@ -341,8 +341,7 @@ private Collection loadWatches(ClusterState clusterState) { } try { - Watch watch = parser.parse(id, true, hit.getSourceRef(), XContentType.JSON); - watch.version(hit.getVersion()); + Watch watch = parser.parse(id, true, hit.getSourceRef(), XContentType.JSON, hit.getSeqNo(), hit.getPrimaryTerm()); if (watch.status().state().isActive()) { watches.add(watch); } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/execution/ExecutionService.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/execution/ExecutionService.java index 60f9c02eb297c..3950b05034eb4 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/execution/ExecutionService.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/execution/ExecutionService.java @@ -282,7 +282,8 @@ record = ctx.abortBeforeExecution(ExecutionState.NOT_EXECUTED_ALREADY_QUEUED, "W if (resp.isExists() == false) { throw new ResourceNotFoundException("watch [{}] does not exist", watchId); } - return parser.parseWithSecrets(watchId, true, resp.getSourceAsBytesRef(), ctx.executionTime(), XContentType.JSON); + return parser.parseWithSecrets(watchId, true, resp.getSourceAsBytesRef(), ctx.executionTime(), XContentType.JSON, + resp.getSeqNo(), resp.getPrimaryTerm()); }); } catch (ResourceNotFoundException e) { String message = "unable to find watch for record [" + ctx.id() + "]"; @@ -353,7 +354,8 @@ public void updateWatchStatus(Watch watch) throws IOException { UpdateRequest updateRequest = new UpdateRequest(Watch.INDEX, Watch.DOC_TYPE, watch.id()); updateRequest.doc(source); - updateRequest.version(watch.version()); + updateRequest.setIfSeqNo(watch.getSourceSeqNo()); + updateRequest.setIfPrimaryTerm(watch.getSourcePrimaryTerm()); try (ThreadContext.StoredContext ignore = stashWithOrigin(client.threadPool().getThreadContext(), WATCHER_ORIGIN)) { client.update(updateRequest).actionGet(indexDefaultTimeout); } catch (DocumentMissingException e) { diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestExecuteWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestExecuteWatchAction.java index 4b79c72d417a5..2c85dca35c0e4 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestExecuteWatchAction.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestExecuteWatchAction.java @@ -50,8 +50,7 @@ public class RestExecuteWatchAction extends WatcherRestHandler implements RestRe WatchField.INPUT.getPreferredName(), WatchField.CONDITION.getPreferredName(), WatchField.ACTIONS.getPreferredName(), WatchField.TRANSFORM.getPreferredName(), WatchField.THROTTLE_PERIOD.getPreferredName(), WatchField.THROTTLE_PERIOD_HUMAN.getPreferredName(), - WatchField.METADATA.getPreferredName(), WatchField.STATUS.getPreferredName(), - WatchField.VERSION.getPreferredName()); + WatchField.METADATA.getPreferredName(), WatchField.STATUS.getPreferredName()); public RestExecuteWatchAction(Settings settings, RestController controller) { super(settings); diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestGetWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestGetWatchAction.java index 1ed6b0d795e91..15cb9612445e1 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestGetWatchAction.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestGetWatchAction.java @@ -9,7 +9,6 @@ import org.apache.logging.log4j.LogManager; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.rest.BytesRestResponse; import org.elasticsearch.rest.RestController; @@ -22,7 +21,6 @@ import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse; import org.elasticsearch.xpack.watcher.rest.WatcherRestHandler; - import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestStatus.NOT_FOUND; import static org.elasticsearch.rest.RestStatus.OK; @@ -50,17 +48,9 @@ protected RestChannelConsumer doPrepareRequest(final RestRequest request, Watche return channel -> client.getWatch(getWatchRequest, new RestBuilderListener(channel) { @Override public RestResponse buildResponse(GetWatchResponse response, XContentBuilder builder) throws Exception { - builder.startObject() - .field("found", response.isFound()) - .field("_id", response.getId()); - if (response.isFound()) { - builder.field("_version", response.getVersion()); - ToXContent.MapParams xContentParams = new ToXContent.MapParams(request.params()); - builder.field("status", response.getStatus(), xContentParams); - builder.field("watch", response.getSource(), xContentParams); - } - builder.endObject(); - + builder.startObject(); + response.toXContent(builder, request); + builder.endObject(); RestStatus status = response.isFound() ? OK : NOT_FOUND; return new BytesRestResponse(status, builder); } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestPutWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestPutWatchAction.java index 31a5162499641..edbfa3cf1fe28 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestPutWatchAction.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestPutWatchAction.java @@ -58,15 +58,13 @@ protected RestChannelConsumer doPrepareRequest(final RestRequest request, Watche PutWatchRequest putWatchRequest = new PutWatchRequest(request.param("id"), request.content(), request.getXContentType()); putWatchRequest.setVersion(request.paramAsLong("version", Versions.MATCH_ANY)); + putWatchRequest.setIfSeqNo(request.paramAsLong("if_seq_no", putWatchRequest.getIfSeqNo())); + putWatchRequest.setIfPrimaryTerm(request.paramAsLong("if_primary_term", putWatchRequest.getIfPrimaryTerm())); putWatchRequest.setActive(request.paramAsBoolean("active", putWatchRequest.isActive())); return channel -> client.putWatch(putWatchRequest, new RestBuilderListener(channel) { @Override public RestResponse buildResponse(PutWatchResponse response, XContentBuilder builder) throws Exception { - builder.startObject() - .field("_id", response.getId()) - .field("_version", response.getVersion()) - .field("created", response.isCreated()) - .endObject(); + response.toXContent(builder, request); RestStatus status = response.isCreated() ? CREATED : OK; return new BytesRestResponse(status, builder); } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/ack/TransportAckWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/ack/TransportAckWatchAction.java index 9db741ecf4773..81788d7af01bf 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/ack/TransportAckWatchAction.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/ack/TransportAckWatchAction.java @@ -7,6 +7,7 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; @@ -16,6 +17,7 @@ import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.routing.Preference; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -49,15 +51,17 @@ public class TransportAckWatchAction extends WatcherTransportActionwrap(getResponse -> { if (getResponse.isExists()) { Watch watch = parser.parseWithSecrets(request.getWatchId(), true, getResponse.getSourceAsBytesRef(), now, - XContentType.JSON); - watch.version(getResponse.getVersion()); + XContentType.JSON, getResponse.getSeqNo(), getResponse.getPrimaryTerm()); watch.status().version(getResponse.getVersion()); listener.onResponse(new ActivateWatchResponse(watch.status())); } else { diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/execute/TransportExecuteWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/execute/TransportExecuteWatchAction.java index eca30cfca8c56..6a25808c40e36 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/execute/TransportExecuteWatchAction.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/execute/TransportExecuteWatchAction.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -86,9 +87,8 @@ protected void doExecute(ExecuteWatchRequest request, ActionListenerwrap(response -> { if (response.isExists()) { - Watch watch = - watchParser.parse(request.getId(), true, response.getSourceAsBytesRef(), request.getXContentType()); - watch.version(response.getVersion()); + Watch watch = watchParser.parse(request.getId(), true, response.getSourceAsBytesRef(), + request.getXContentType(), response.getSeqNo(), response.getPrimaryTerm()); watch.status().version(response.getVersion()); executeWatch(request, listener, watch, true); } else { @@ -99,7 +99,7 @@ protected void doExecute(ExecuteWatchRequest request, ActionListener listener) { try { DateTime now = new DateTime(clock.millis(), UTC); - boolean isUpdate = request.getVersion() > 0; - Watch watch = parser.parseWithSecrets(request.getId(), false, request.getSource(), now, request.xContentType(), isUpdate); + boolean isUpdate = request.getVersion() > 0 || request.getIfSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO; + Watch watch = parser.parseWithSecrets(request.getId(), false, request.getSource(), now, request.xContentType(), + isUpdate, request.getIfSeqNo(), request.getIfPrimaryTerm()); watch.setState(request.isActive(), now); // ensure we only filter for the allowed headers @@ -92,14 +94,20 @@ protected void doExecute(PutWatchRequest request, ActionListenerwrap(response -> { boolean created = response.getResult() == DocWriteResponse.Result.CREATED; - listener.onResponse(new PutWatchResponse(response.getId(), response.getVersion(), created)); + listener.onResponse(new PutWatchResponse(response.getId(), response.getVersion(), + response.getSeqNo(), response.getPrimaryTerm(), created)); }, listener::onFailure), client::update); } else { @@ -109,7 +117,8 @@ protected void doExecute(PutWatchRequest request, ActionListenerwrap(response -> { boolean created = response.getResult() == DocWriteResponse.Result.CREATED; - listener.onResponse(new PutWatchResponse(response.getId(), response.getVersion(), created)); + listener.onResponse(new PutWatchResponse(response.getId(), response.getVersion(), + response.getSeqNo(), response.getPrimaryTerm(), created)); }, listener::onFailure), client::index); } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/watch/WatchParser.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/watch/WatchParser.java index ab316c3dd1001..55f65943a1381 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/watch/WatchParser.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/watch/WatchParser.java @@ -10,7 +10,6 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -74,13 +73,15 @@ public WatchParser(TriggerService triggerService, ActionRegistry actionRegistry, this.defaultActions = Collections.emptyList(); } - public Watch parse(String name, boolean includeStatus, BytesReference source, XContentType xContentType) throws IOException { - return parse(name, includeStatus, false, source, new DateTime(clock.millis(), UTC), xContentType, false); + public Watch parse(String name, boolean includeStatus, BytesReference source, XContentType xContentType, + long sourceSeqNo, long sourcePrimaryTerm) throws IOException { + return parse(name, includeStatus, false, source, new DateTime(clock.millis(), UTC), xContentType, false, + sourceSeqNo, sourcePrimaryTerm); } public Watch parse(String name, boolean includeStatus, BytesReference source, DateTime now, - XContentType xContentType) throws IOException { - return parse(name, includeStatus, false, source, now, xContentType, false); + XContentType xContentType, long sourceSeqNo, long sourcePrimaryTerm) throws IOException { + return parse(name, includeStatus, false, source, now, xContentType, false, sourceSeqNo, sourcePrimaryTerm); } /** @@ -95,17 +96,20 @@ public Watch parse(String name, boolean includeStatus, BytesReference source, Da * */ public Watch parseWithSecrets(String id, boolean includeStatus, BytesReference source, DateTime now, - XContentType xContentType, boolean allowRedactedPasswords) throws IOException { - return parse(id, includeStatus, true, source, now, xContentType, allowRedactedPasswords); + XContentType xContentType, boolean allowRedactedPasswords, long sourceSeqNo, long sourcePrimaryTerm + ) throws IOException { + return parse(id, includeStatus, true, source, now, xContentType, allowRedactedPasswords, sourceSeqNo, sourcePrimaryTerm); } + public Watch parseWithSecrets(String id, boolean includeStatus, BytesReference source, DateTime now, - XContentType xContentType) throws IOException { - return parse(id, includeStatus, true, source, now, xContentType, false); + XContentType xContentType, long sourceSeqNo, long sourcePrimaryTerm) throws IOException { + return parse(id, includeStatus, true, source, now, xContentType, false, sourceSeqNo, sourcePrimaryTerm); } private Watch parse(String id, boolean includeStatus, boolean withSecrets, BytesReference source, DateTime now, - XContentType xContentType, boolean allowRedactedPasswords) throws IOException { + XContentType xContentType, boolean allowRedactedPasswords, long sourceSeqNo, long sourcePrimaryTerm) + throws IOException { if (logger.isTraceEnabled()) { logger.trace("parsing watch [{}] ", source.utf8ToString()); } @@ -115,13 +119,14 @@ private Watch parse(String id, boolean includeStatus, boolean withSecrets, Bytes LoggingDeprecationHandler.INSTANCE, stream), now, withSecrets ? cryptoService : null, allowRedactedPasswords)) { parser.nextToken(); - return parse(id, includeStatus, parser); + return parse(id, includeStatus, parser, sourceSeqNo, sourcePrimaryTerm); } catch (IOException ioe) { throw ioException("could not parse watch [{}]", ioe, id); } } - public Watch parse(String id, boolean includeStatus, WatcherXContentParser parser) throws IOException { + public Watch parse(String id, boolean includeStatus, WatcherXContentParser parser, long sourceSeqNo, long sourcePrimaryTerm) + throws IOException { Trigger trigger = null; ExecutableInput input = defaultInput; ExecutableCondition condition = defaultCondition; @@ -130,7 +135,6 @@ public Watch parse(String id, boolean includeStatus, WatcherXContentParser parse TimeValue throttlePeriod = null; Map metatdata = null; WatchStatus status = null; - long version = Versions.MATCH_ANY; String currentFieldName = null; XContentParser.Token token; @@ -163,8 +167,6 @@ public Watch parse(String id, boolean includeStatus, WatcherXContentParser parse actions = actionRegistry.parseActions(id, parser); } else if (WatchField.METADATA.match(currentFieldName, parser.getDeprecationHandler())) { metatdata = parser.map(); - } else if (WatchField.VERSION.match(currentFieldName, parser.getDeprecationHandler())) { - version = parser.longValue(); } else if (WatchField.STATUS.match(currentFieldName, parser.getDeprecationHandler())) { if (includeStatus) { status = WatchStatus.parse(id, parser); @@ -198,6 +200,7 @@ public Watch parse(String id, boolean includeStatus, WatcherXContentParser parse } - return new Watch(id, trigger, input, condition, transform, throttlePeriod, actions, metatdata, status, version); + return new Watch( + id, trigger, input, condition, transform, throttlePeriod, actions, metatdata, status, sourceSeqNo, sourcePrimaryTerm); } } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherIndexingListenerTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherIndexingListenerTests.java index f351ed2e154ed..7a37f71e22f28 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherIndexingListenerTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherIndexingListenerTests.java @@ -63,6 +63,7 @@ import static org.hamcrest.core.Is.is; import static org.joda.time.DateTimeZone.UTC; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; import static org.mockito.Matchers.anyObject; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; @@ -134,13 +135,13 @@ public void testPreIndex() throws Exception { boolean watchActive = randomBoolean(); boolean isNewWatch = randomBoolean(); Watch watch = mockWatch("_id", watchActive, isNewWatch); - when(parser.parseWithSecrets(anyObject(), eq(true), anyObject(), anyObject(), anyObject())).thenReturn(watch); + when(parser.parseWithSecrets(anyObject(), eq(true), anyObject(), anyObject(), anyObject(), anyLong(), anyLong())).thenReturn(watch); Engine.Index returnedOperation = listener.preIndex(shardId, operation); assertThat(returnedOperation, is(operation)); DateTime now = new DateTime(clock.millis(), UTC); - verify(parser).parseWithSecrets(eq(operation.id()), eq(true), eq(BytesArray.EMPTY), eq(now), anyObject()); + verify(parser).parseWithSecrets(eq(operation.id()), eq(true), eq(BytesArray.EMPTY), eq(now), anyObject(), anyLong(), anyLong()); if (isNewWatch) { if (watchActive) { @@ -162,7 +163,7 @@ public void testPreIndexWatchGetsOnlyTriggeredOnceAcrossAllShards() throws Excep when(shardId.getIndexName()).thenReturn(Watch.INDEX); when(operation.type()).thenReturn(Watch.DOC_TYPE); - when(parser.parseWithSecrets(anyObject(), eq(true), anyObject(), anyObject(), anyObject())).thenReturn(watch); + when(parser.parseWithSecrets(anyObject(), eq(true), anyObject(), anyObject(), anyObject(), anyLong(), anyLong())).thenReturn(watch); for (int idx = 0; idx < totalShardCount; idx++) { final Map localShards = new HashMap<>(); @@ -207,7 +208,7 @@ public void testPreIndexCheckParsingException() throws Exception { when(operation.id()).thenReturn(id); when(operation.source()).thenReturn(BytesArray.EMPTY); when(shardId.getIndexName()).thenReturn(Watch.INDEX); - when(parser.parseWithSecrets(anyObject(), eq(true), anyObject(), anyObject(), anyObject())) + when(parser.parseWithSecrets(anyObject(), eq(true), anyObject(), anyObject(), anyObject(), anyLong(), anyLong())) .thenThrow(new IOException("self thrown")); ElasticsearchParseException exc = expectThrows(ElasticsearchParseException.class, diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java index ab9fff7dbe587..96186905c26ad 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java @@ -70,6 +70,7 @@ import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -194,7 +195,7 @@ void stopExecutor() { Watch watch = mock(Watch.class); when(watchStatus.state()).thenReturn(state); when(watch.status()).thenReturn(watchStatus); - when(parser.parse(eq(id), eq(true), any(), eq(XContentType.JSON))).thenReturn(watch); + when(parser.parse(eq(id), eq(true), any(), eq(XContentType.JSON), anyLong(), anyLong())).thenReturn(watch); } SearchHits searchHits = new SearchHits(hits, new TotalHits(count, TotalHits.Relation.EQUAL_TO), 1.0f); SearchResponseSections sections = new SearchResponseSections(searchHits, null, null, false, false, null, 1); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java index 287b3976dea3a..5f2b807f10757 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java @@ -101,6 +101,7 @@ import static org.joda.time.DateTime.now; import static org.joda.time.DateTimeZone.UTC; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; import static org.mockito.Matchers.anyObject; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; @@ -166,7 +167,7 @@ public void testExecute() throws Exception { DateTime now = new DateTime(clock.millis()); ScheduleTriggerEvent event = new ScheduleTriggerEvent("_id", now, now); TriggeredExecutionContext context = new TriggeredExecutionContext(watch.id(), now, event, timeValueSeconds(5)); - when(parser.parseWithSecrets(eq(watch.id()), eq(true), any(), any(), any())).thenReturn(watch); + when(parser.parseWithSecrets(eq(watch.id()), eq(true), any(), any(), any(), anyLong(), anyLong())).thenReturn(watch); Condition.Result conditionResult = InternalAlwaysCondition.RESULT_INSTANCE; ExecutableCondition condition = mock(ExecutableCondition.class); @@ -270,7 +271,7 @@ public void testExecuteFailedInput() throws Exception { DateTime now = new DateTime(clock.millis()); ScheduleTriggerEvent event = new ScheduleTriggerEvent("_id", now, now); TriggeredExecutionContext context = new TriggeredExecutionContext(watch.id(), now, event, timeValueSeconds(5)); - when(parser.parseWithSecrets(eq(watch.id()), eq(true), any(), any(), any())).thenReturn(watch); + when(parser.parseWithSecrets(eq(watch.id()), eq(true), any(), any(), any(), anyLong(), anyLong())).thenReturn(watch); input = mock(ExecutableInput.class); Input.Result inputResult = mock(Input.Result.class); @@ -339,7 +340,7 @@ public void testExecuteFailedCondition() throws Exception { DateTime now = new DateTime(clock.millis()); ScheduleTriggerEvent event = new ScheduleTriggerEvent("_id", now, now); TriggeredExecutionContext context = new TriggeredExecutionContext(watch.id(), now, event, timeValueSeconds(5)); - when(parser.parseWithSecrets(eq(watch.id()), eq(true), any(), any(), any())).thenReturn(watch); + when(parser.parseWithSecrets(eq(watch.id()), eq(true), any(), any(), any(), anyLong(), anyLong())).thenReturn(watch); ExecutableCondition condition = mock(ExecutableCondition.class); Condition.Result conditionResult = mock(Condition.Result.class); @@ -404,7 +405,7 @@ public void testExecuteFailedWatchTransform() throws Exception { DateTime now = new DateTime(clock.millis()); ScheduleTriggerEvent event = new ScheduleTriggerEvent("_id", now, now); TriggeredExecutionContext context = new TriggeredExecutionContext(watch.id(), now, event, timeValueSeconds(5)); - when(parser.parseWithSecrets(eq(watch.id()), eq(true), any(), any(), any())).thenReturn(watch); + when(parser.parseWithSecrets(eq(watch.id()), eq(true), any(), any(), any(), anyLong(), anyLong())).thenReturn(watch); Condition.Result conditionResult = InternalAlwaysCondition.RESULT_INSTANCE; ExecutableCondition condition = mock(ExecutableCondition.class); @@ -464,7 +465,7 @@ public void testExecuteFailedActionTransform() throws Exception { GetResponse getResponse = mock(GetResponse.class); when(getResponse.isExists()).thenReturn(true); mockGetWatchResponse(client, "_id", getResponse); - when(parser.parseWithSecrets(eq(watch.id()), eq(true), any(), any(), any())).thenReturn(watch); + when(parser.parseWithSecrets(eq(watch.id()), eq(true), any(), any(), any(), anyLong(), anyLong())).thenReturn(watch); DateTime now = new DateTime(clock.millis()); ScheduleTriggerEvent event = new ScheduleTriggerEvent("_id", now, now); @@ -838,7 +839,7 @@ public void testThatTriggeredWatchDeletionWorksOnExecutionRejection() throws Exc when(getResponse.isExists()).thenReturn(true); when(getResponse.getId()).thenReturn("foo"); mockGetWatchResponse(client, "foo", getResponse); - when(parser.parseWithSecrets(eq("foo"), eq(true), any(), any(), any())).thenReturn(watch); + when(parser.parseWithSecrets(eq("foo"), eq(true), any(), any(), any(), anyLong(), anyLong())).thenReturn(watch); // execute needs to fail as well as storing the history doThrow(new EsRejectedExecutionException()).when(executor).execute(any()); @@ -971,7 +972,7 @@ public void testWatchInactive() throws Exception { GetResponse getResponse = mock(GetResponse.class); when(getResponse.isExists()).thenReturn(true); mockGetWatchResponse(client, "_id", getResponse); - when(parser.parseWithSecrets(eq(watch.id()), eq(true), any(), any(), any())).thenReturn(watch); + when(parser.parseWithSecrets(eq(watch.id()), eq(true), any(), any(), any(), anyLong(), anyLong())).thenReturn(watch); WatchRecord.MessageWatchRecord record = mock(WatchRecord.MessageWatchRecord.class); when(record.state()).thenReturn(ExecutionState.EXECUTION_NOT_NEEDED); @@ -1017,7 +1018,7 @@ public void testQueuedWatches() throws Exception { public void testUpdateWatchStatusDoesNotUpdateState() throws Exception { WatchStatus status = new WatchStatus(DateTime.now(UTC), Collections.emptyMap()); Watch watch = new Watch("_id", new ManualTrigger(), new ExecutableNoneInput(), InternalAlwaysCondition.INSTANCE, null, null, - Collections.emptyList(), null, status, 1L); + Collections.emptyList(), null, status, 1L, 1L); final AtomicBoolean assertionsTriggered = new AtomicBoolean(false); doAnswer(invocation -> { diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherTestUtils.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherTestUtils.java index e22d3d6f0837c..f2a9941ec35ed 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherTestUtils.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherTestUtils.java @@ -55,7 +55,6 @@ import org.joda.time.DateTime; import javax.mail.internet.AddressException; - import java.io.IOException; import java.util.ArrayList; import java.util.Collections; @@ -134,7 +133,7 @@ public static WatchExecutionContext createWatchExecutionContext() throws Excepti null, new ArrayList<>(), null, - new WatchStatus(new DateTime(0, UTC), emptyMap()), 1L); + new WatchStatus(new DateTime(0, UTC), emptyMap()), 1L, 1L); TriggeredExecutionContext context = new TriggeredExecutionContext(watch.id(), new DateTime(0, UTC), new ScheduleTriggerEvent(watch.id(), new DateTime(0, UTC), new DateTime(0, UTC)), @@ -181,7 +180,7 @@ public static Watch createTestWatch(String watchName, Client client, HttpClient new TimeValue(0), actions, Collections.singletonMap("foo", "bar"), - new WatchStatus(now, statuses), 1L); + new WatchStatus(now, statuses), 1L, 1L); } public static SearchType getRandomSupportedSearchType() { diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/bench/ScheduleEngineTriggerBenchmark.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/bench/ScheduleEngineTriggerBenchmark.java index d1b8963950dd8..183e80bf2528d 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/bench/ScheduleEngineTriggerBenchmark.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/bench/ScheduleEngineTriggerBenchmark.java @@ -57,7 +57,7 @@ public static void main(String[] args) throws Exception { List watches = new ArrayList<>(numWatches); for (int i = 0; i < numWatches; i++) { watches.add(new Watch("job_" + i, new ScheduleTrigger(interval(interval + "s")), new ExecutableNoneInput(), - InternalAlwaysCondition.INSTANCE, null, null, Collections.emptyList(), null, null, 1L)); + InternalAlwaysCondition.INSTANCE, null, null, Collections.emptyList(), null, null, 1L, 1L)); } ScheduleRegistry scheduleRegistry = new ScheduleRegistry(emptySet()); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatchAckTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatchAckTests.java index cb8fc9edb6f14..c1a8623b44719 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatchAckTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatchAckTests.java @@ -110,7 +110,8 @@ public void testAckSingleAction() throws Exception { GetWatchResponse getWatchResponse = watcherClient().prepareGetWatch("_id").get(); assertThat(getWatchResponse.isFound(), is(true)); - Watch parsedWatch = watchParser().parse(getWatchResponse.getId(), true, getWatchResponse.getSource().getBytes(), XContentType.JSON); + Watch parsedWatch = watchParser().parse(getWatchResponse.getId(), true, getWatchResponse.getSource().getBytes(), + XContentType.JSON, getWatchResponse.getSeqNo(), getWatchResponse.getPrimaryTerm()); assertThat(parsedWatch.status().actionStatus("_a1").ackStatus().state(), is(ActionStatus.AckStatus.State.AWAITS_SUCCESSFUL_EXECUTION)); assertThat(parsedWatch.status().actionStatus("_a2").ackStatus().state(), @@ -178,7 +179,8 @@ public void testAckAllActions() throws Exception { GetWatchResponse getWatchResponse = watcherClient().prepareGetWatch("_id").get(); assertThat(getWatchResponse.isFound(), is(true)); - Watch parsedWatch = watchParser().parse(getWatchResponse.getId(), true, getWatchResponse.getSource().getBytes(), XContentType.JSON); + Watch parsedWatch = watchParser().parse(getWatchResponse.getId(), true, + getWatchResponse.getSource().getBytes(), XContentType.JSON, getWatchResponse.getSeqNo(), getWatchResponse.getPrimaryTerm()); assertThat(parsedWatch.status().actionStatus("_a1").ackStatus().state(), is(ActionStatus.AckStatus.State.AWAITS_SUCCESSFUL_EXECUTION)); assertThat(parsedWatch.status().actionStatus("_a2").ackStatus().state(), @@ -220,7 +222,8 @@ public void testAckWithRestart() throws Exception { refresh(); GetResponse getResponse = client().get(new GetRequest(Watch.INDEX, Watch.DOC_TYPE, "_name")).actionGet(); - Watch indexedWatch = watchParser().parse("_name", true, getResponse.getSourceAsBytesRef(), XContentType.JSON); + Watch indexedWatch = watchParser().parse("_name", true, getResponse.getSourceAsBytesRef(), XContentType.JSON, + getResponse.getSeqNo(), getResponse.getPrimaryTerm()); assertThat(watchResponse.getStatus().actionStatus("_id").ackStatus().state(), equalTo(indexedWatch.status().actionStatus("_id").ackStatus().state())); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/ack/TransportAckWatchActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/ack/TransportAckWatchActionTests.java index 57c189d328e8a..956cce95f84c2 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/ack/TransportAckWatchActionTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/ack/TransportAckWatchActionTests.java @@ -39,6 +39,7 @@ import java.util.concurrent.ExecutionException; import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; +import static org.elasticsearch.test.ClusterServiceUtils.createClusterService; import static org.hamcrest.Matchers.is; import static org.mockito.Matchers.anyObject; import static org.mockito.Matchers.eq; @@ -61,7 +62,7 @@ public void setupAction() { client = mock(Client.class); when(client.threadPool()).thenReturn(threadPool); action = new TransportAckWatchAction(transportService, new ActionFilters(Collections.emptySet()), - Clock.systemUTC(), new XPackLicenseState(Settings.EMPTY), watchParser, client); + Clock.systemUTC(), new XPackLicenseState(Settings.EMPTY), watchParser, client, createClusterService(threadPool)); } public void testWatchNotFound() { diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/put/TransportPutWatchActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/put/TransportPutWatchActionTests.java index 5ae7c19f625c5..411bee5e2f9d9 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/put/TransportPutWatchActionTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/put/TransportPutWatchActionTests.java @@ -36,6 +36,7 @@ import static org.hamcrest.Matchers.nullValue; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Matchers.anyLong; import static org.mockito.Matchers.anyObject; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doAnswer; @@ -57,7 +58,8 @@ public void setupAction() throws Exception { TransportService transportService = mock(TransportService.class); WatchParser parser = mock(WatchParser.class); - when(parser.parseWithSecrets(eq("_id"), eq(false), anyObject(), anyObject(), anyObject(), anyBoolean())).thenReturn(watch); + when(parser.parseWithSecrets(eq("_id"), eq(false), anyObject(), anyObject(), anyObject(), anyBoolean(), anyLong(), anyLong())) + .thenReturn(watch); Client client = mock(Client.class); when(client.threadPool()).thenReturn(threadPool); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/schedule/engine/TickerScheduleEngineTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/schedule/engine/TickerScheduleEngineTests.java index 898ae8ac9aa55..0f6ca200cb4ab 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/schedule/engine/TickerScheduleEngineTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/schedule/engine/TickerScheduleEngineTests.java @@ -5,8 +5,8 @@ */ package org.elasticsearch.xpack.watcher.trigger.schedule.engine; -import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.watcher.trigger.TriggerEvent; import org.elasticsearch.xpack.core.watcher.watch.ClockMock; @@ -273,6 +273,6 @@ public void testAddOnlyWithNewSchedule() { private Watch createWatch(String name, Schedule schedule) { return new Watch(name, new ScheduleTrigger(schedule), new ExecutableNoneInput(), InternalAlwaysCondition.INSTANCE, null, null, - Collections.emptyList(), null, null, Versions.MATCH_ANY); + Collections.emptyList(), null, null, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM); } } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java index cb32c3eebec09..95a63c97f4dfe 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java @@ -208,13 +208,16 @@ public void testParserSelfGenerated() throws Exception { TimeValue throttlePeriod = randomBoolean() ? null : TimeValue.timeValueSeconds(randomIntBetween(5, 10000)); - Watch watch = new Watch("_name", trigger, input, condition, transform, throttlePeriod, actions, metadata, watchStatus, 1L); + final long sourceSeqNo = randomNonNegativeLong(); + final long sourcePrimaryTerm = randomLongBetween(1, 200); + Watch watch = new Watch("_name", trigger, input, condition, transform, throttlePeriod, actions, metadata, watchStatus, + sourceSeqNo, sourcePrimaryTerm); BytesReference bytes = BytesReference.bytes(jsonBuilder().value(watch)); logger.info("{}", bytes.utf8ToString()); WatchParser watchParser = new WatchParser(triggerService, actionRegistry, inputRegistry, null, clock); - Watch parsedWatch = watchParser.parse("_name", includeStatus, bytes, XContentType.JSON); + Watch parsedWatch = watchParser.parse("_name", includeStatus, bytes, XContentType.JSON, sourceSeqNo, sourcePrimaryTerm); if (includeStatus) { assertThat(parsedWatch.status(), equalTo(watchStatus)); @@ -227,6 +230,8 @@ public void testParserSelfGenerated() throws Exception { } assertThat(parsedWatch.metadata(), equalTo(metadata)); assertThat(parsedWatch.actions(), equalTo(actions)); + assertThat(parsedWatch.getSourceSeqNo(), equalTo(sourceSeqNo)); + assertThat(parsedWatch.getSourcePrimaryTerm(), equalTo(sourcePrimaryTerm)); } public void testThatBothStatusFieldsCanBeRead() throws Exception { @@ -256,7 +261,7 @@ public Trigger parseTrigger(String jobName, XContentParser parser) throws IOExce WatchParser watchParser = new WatchParser(triggerService, actionRegistry, inputRegistry, null, clock); XContentBuilder builder = jsonBuilder().startObject().startObject("trigger").endObject().field("status", watchStatus).endObject(); - Watch watch = watchParser.parse("foo", true, BytesReference.bytes(builder), XContentType.JSON); + Watch watch = watchParser.parse("foo", true, BytesReference.bytes(builder), XContentType.JSON, 1L, 1L); assertThat(watch.status().state().getTimestamp().getMillis(), is(clock.millis())); for (ActionWrapper action : actions) { assertThat(watch.status().actionStatus(action.id()), is(actionsStatuses.get(action.id()))); @@ -284,7 +289,7 @@ public void testParserBadActions() throws Exception { .endObject(); WatchParser watchParser = new WatchParser(triggerService, actionRegistry, inputRegistry, null, clock); try { - watchParser.parse("failure", false, BytesReference.bytes(jsonBuilder), XContentType.JSON); + watchParser.parse("failure", false, BytesReference.bytes(jsonBuilder), XContentType.JSON, 1L, 1L); fail("This watch should fail to parse as actions is an array"); } catch (ElasticsearchParseException pe) { assertThat(pe.getMessage().contains("could not parse actions for watch [failure]"), is(true)); @@ -309,7 +314,7 @@ public void testParserDefaults() throws Exception { .endObject(); builder.endObject(); WatchParser watchParser = new WatchParser(triggerService, actionRegistry, inputRegistry, null, Clock.systemUTC()); - Watch watch = watchParser.parse("failure", false, BytesReference.bytes(builder), XContentType.JSON); + Watch watch = watchParser.parse("failure", false, BytesReference.bytes(builder), XContentType.JSON, 1L, 1L); assertThat(watch, notNullValue()); assertThat(watch.trigger(), instanceOf(ScheduleTrigger.class)); assertThat(watch.input(), instanceOf(ExecutableNoneInput.class)); @@ -375,7 +380,7 @@ public void testParseWatch_verifyScriptLangDefault() throws Exception { builder.endObject(); // parse in default mode: - Watch watch = watchParser.parse("_id", false, BytesReference.bytes(builder), XContentType.JSON); + Watch watch = watchParser.parse("_id", false, BytesReference.bytes(builder), XContentType.JSON, 1L, 1L); assertThat(((ScriptCondition) watch.condition()).getScript().getLang(), equalTo(Script.DEFAULT_SCRIPT_LANG)); WatcherSearchTemplateRequest request = ((SearchInput) watch.input().input()).getRequest(); SearchRequest searchRequest = searchTemplateService.toSearchRequest(request); @@ -394,7 +399,7 @@ public void testParseWatchWithoutInput() throws Exception { builder.endObject(); WatchParser parser = createWatchparser(); - Watch watch = parser.parse("_id", false, BytesReference.bytes(builder), XContentType.JSON); + Watch watch = parser.parse("_id", false, BytesReference.bytes(builder), XContentType.JSON, 1L, 1L); assertThat(watch, is(notNullValue())); assertThat(watch.input().type(), is(NoneInput.TYPE)); } @@ -410,7 +415,7 @@ public void testParseWatchWithoutAction() throws Exception { builder.endObject(); WatchParser parser = createWatchparser(); - Watch watch = parser.parse("_id", false, BytesReference.bytes(builder), XContentType.JSON); + Watch watch = parser.parse("_id", false, BytesReference.bytes(builder), XContentType.JSON, 1L, 1L); assertThat(watch, is(notNullValue())); assertThat(watch.actions(), hasSize(0)); } @@ -429,7 +434,7 @@ public void testParseWatchWithoutTriggerDoesNotWork() throws Exception { WatchParser parser = createWatchparser(); ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, - () -> parser.parse("_id", false, BytesReference.bytes(builder), XContentType.JSON)); + () -> parser.parse("_id", false, BytesReference.bytes(builder), XContentType.JSON, 1L, 1L)); assertThat(e.getMessage(), is("could not parse watch [_id]. missing required field [trigger]")); } } From a6a534f1f0b19075dc26455c6f2bd2a692628324 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 30 Jan 2019 20:34:27 -0500 Subject: [PATCH 079/100] Reenable BWC testing after retention lease stats (#38062) This commit adjusts the BWC version on retention leases in stats, so with this we also reenable BWC testing. --- build.gradle | 4 ++-- .../elasticsearch/action/admin/indices/stats/ShardStats.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index 164add875eb3b..4bd211a12b3b0 100644 --- a/build.gradle +++ b/build.gradle @@ -159,8 +159,8 @@ task verifyVersions { * the enabled state of every bwc task. It should be set back to true * after the backport of the backcompat code is complete. */ -final boolean bwc_tests_enabled = false -final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/37991" /* place a PR link here when committing bwc changes */ +final boolean bwc_tests_enabled = true +final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */ if (bwc_tests_enabled == false) { if (bwc_tests_disabled_issue.isEmpty()) { throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false") diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java index 9297447695a61..15173e581a738 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java @@ -130,7 +130,7 @@ public void readFrom(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { seqNoStats = in.readOptionalWriteable(SeqNoStats::new); } - if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + if (in.getVersion().onOrAfter(Version.V_6_7_0)) { retentionLeaseStats = in.readOptionalWriteable(RetentionLeaseStats::new); } } @@ -146,7 +146,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { out.writeOptionalWriteable(seqNoStats); } - if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + if (out.getVersion().onOrAfter(Version.V_6_7_0)) { out.writeOptionalWriteable(retentionLeaseStats); } } From 89bffc25de064dc20e5d1cd9d4d509bfb61d7612 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 30 Jan 2019 20:37:52 -0500 Subject: [PATCH 080/100] Mute failing date index name processor test This test is repeatedly failing, so this commit mutes it. Relates #38067 --- .../elasticsearch/ingest/common/DateIndexNameProcessorTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java index 760e48f31ff29..3ac885d8680e9 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java @@ -79,6 +79,7 @@ public void testUnix()throws Exception { assertThat(document.getSourceAndMetadata().get("_index"), equalTo("")); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/38067") public void testTemplatedFields() throws Exception { String indexNamePrefix = randomAlphaOfLength(10); String dateRounding = randomFrom("y", "M", "w", "d", "h", "m", "s"); From f5398d65119e7ea851fca9e806dc7677a987d11e Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 31 Jan 2019 00:57:27 -0500 Subject: [PATCH 081/100] Mute testRetentionLeasesSyncOnExpiration Tracked at #37963 --- .../java/org/elasticsearch/index/seqno/RetentionLeaseSyncIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncIT.java b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncIT.java index f2819bfe161eb..98367c0e97188 100644 --- a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncIT.java +++ b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncIT.java @@ -96,6 +96,7 @@ public void testRetentionLeasesSyncedOnAdd() throws Exception { } } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37963") public void testRetentionLeasesSyncOnExpiration() throws Exception { final int numberOfReplicas = 2 - scaledRandomIntBetween(0, 2); internalCluster().ensureAtLeastNumDataNodes(1 + numberOfReplicas); From a8596de31fff2372fadd3f48dd97c6d674f2eb58 Mon Sep 17 00:00:00 2001 From: Tim Vernum Date: Thu, 31 Jan 2019 18:06:05 +1100 Subject: [PATCH 082/100] Introduce ssl settings to reindex from remote (#37527) Adds reindex.ssl.* settings for reindex from remote. This uses the ssl-config/ internal library to parse and load SSL configuration and files. This is applied when using the low level rest client to connect to a remote ES node Relates: #37287 Resolves: #29755 --- build.gradle | 3 +- modules/reindex/build.gradle | 6 + .../AbstractAsyncBulkByScrollAction.java | 11 +- .../reindex/AsyncDeleteByQueryAction.java | 10 +- .../index/reindex/ReindexPlugin.java | 24 +- .../index/reindex/ReindexSslConfig.java | 161 +++++++++++++ .../reindex/TransportDeleteByQueryAction.java | 2 +- .../index/reindex/TransportReindexAction.java | 21 +- .../reindex/TransportUpdateByQueryAction.java | 10 +- ...yncBulkByScrollActionMetadataTestCase.java | 2 +- ...AsyncBulkByScrollActionScriptTestCase.java | 4 +- .../reindex/AsyncBulkByScrollActionTests.java | 22 +- ...ReindexFromRemoteBuildRestClientTests.java | 16 +- .../index/reindex/ReindexMetadataTests.java | 4 +- .../reindex/ReindexRestClientSslTests.java | 213 ++++++++++++++++++ .../index/reindex/ReindexScriptTests.java | 13 +- .../reindex/UpdateByQueryMetadataTests.java | 2 +- .../reindex/UpdateByQueryWithScriptTests.java | 9 +- .../elasticsearch/index/reindex/README.txt | 25 ++ .../org/elasticsearch/index/reindex/ca.p12 | Bin 0 -> 2527 bytes .../org/elasticsearch/index/reindex/ca.pem | 25 ++ .../index/reindex/client/client.crt | 19 ++ .../index/reindex/client/client.key | 30 +++ .../elasticsearch/index/reindex/http/http.crt | 22 ++ .../elasticsearch/index/reindex/http/http.key | 30 +++ 25 files changed, 646 insertions(+), 38 deletions(-) create mode 100644 modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexSslConfig.java create mode 100644 modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexRestClientSslTests.java create mode 100644 modules/reindex/src/test/resources/org/elasticsearch/index/reindex/README.txt create mode 100644 modules/reindex/src/test/resources/org/elasticsearch/index/reindex/ca.p12 create mode 100644 modules/reindex/src/test/resources/org/elasticsearch/index/reindex/ca.pem create mode 100644 modules/reindex/src/test/resources/org/elasticsearch/index/reindex/client/client.crt create mode 100644 modules/reindex/src/test/resources/org/elasticsearch/index/reindex/client/client.key create mode 100644 modules/reindex/src/test/resources/org/elasticsearch/index/reindex/http/http.crt create mode 100644 modules/reindex/src/test/resources/org/elasticsearch/index/reindex/http/http.key diff --git a/build.gradle b/build.gradle index 4bd211a12b3b0..e5bc1ab3ba986 100644 --- a/build.gradle +++ b/build.gradle @@ -201,7 +201,7 @@ allprojects { } /* Sets up the dependencies that we build as part of this project but - register as thought they were external to resolve internally. We register + register as though they were external to resolve internally. We register them as external dependencies so the build plugin that we use can be used to build elasticsearch plugins outside of the elasticsearch source tree. */ ext.projectSubstitutions = [ @@ -214,6 +214,7 @@ allprojects { "org.elasticsearch:elasticsearch-x-content:${version}": ':libs:x-content', "org.elasticsearch:elasticsearch-geo:${version}": ':libs:elasticsearch-geo', "org.elasticsearch:elasticsearch-secure-sm:${version}": ':libs:secure-sm', + "org.elasticsearch:elasticsearch-ssl-config:${version}": ':libs:elasticsearch-ssl-config', "org.elasticsearch.client:elasticsearch-rest-client:${version}": ':client:rest', "org.elasticsearch.client:elasticsearch-rest-client-sniffer:${version}": ':client:sniffer', "org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}": ':client:rest-high-level', diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle index d10d8c34a0c81..36f327a5b6c30 100644 --- a/modules/reindex/build.gradle +++ b/modules/reindex/build.gradle @@ -56,6 +56,7 @@ unitTest { dependencies { compile "org.elasticsearch.client:elasticsearch-rest-client:${version}" + compile "org.elasticsearch:elasticsearch-ssl-config:${version}" // for http - testing reindex from remote testCompile project(path: ':modules:transport-netty4', configuration: 'runtime') // for parent/child testing @@ -71,6 +72,11 @@ thirdPartyAudit.ignoreMissingClasses ( 'org.apache.log.Logger', ) +forbiddenPatterns { + // PKCS#12 file are not UTF-8 + exclude '**/*.p12' +} + // Support for testing reindex-from-remote against old Elaticsearch versions configurations { oldesFixture diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java index 9617d2a3774da..a4b28e6c598d3 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java @@ -34,6 +34,7 @@ import org.elasticsearch.action.bulk.Retry; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.client.ParentTaskAssigningClient; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; @@ -82,13 +83,15 @@ * Abstract base for scrolling across a search and executing bulk actions on all results. All package private methods are package private so * their tests can use them. Most methods run in the listener thread pool because the are meant to be fast and don't expect to block. */ -public abstract class AbstractAsyncBulkByScrollAction> { +public abstract class AbstractAsyncBulkByScrollAction, + Action extends TransportAction> { + protected final Logger logger; protected final BulkByScrollTask task; protected final WorkerBulkByScrollTaskState worker; protected final ThreadPool threadPool; - protected final ScriptService scriptService; + protected final Action mainAction; /** * The request for this action. Named mainRequest because we create lots of request variables all representing child * requests of this mainRequest. @@ -112,7 +115,7 @@ public abstract class AbstractAsyncBulkByScrollAction listener) { this.task = task; @@ -124,7 +127,7 @@ public AbstractAsyncBulkByScrollAction(BulkByScrollTask task, boolean needsSourc this.logger = logger; this.client = client; this.threadPool = threadPool; - this.scriptService = scriptService; + this.mainAction = mainAction; this.mainRequest = mainRequest; this.listener = listener; BackoffPolicy backoffPolicy = buildBackoffPolicy(); diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AsyncDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AsyncDeleteByQueryAction.java index 5026ec0f79c57..b317ea06d9f35 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AsyncDeleteByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AsyncDeleteByQueryAction.java @@ -31,22 +31,22 @@ /** * Implementation of delete-by-query using scrolling and bulk. */ -public class AsyncDeleteByQueryAction extends AbstractAsyncBulkByScrollAction { +public class AsyncDeleteByQueryAction extends AbstractAsyncBulkByScrollAction { + private final boolean useSeqNoForCAS; public AsyncDeleteByQueryAction(BulkByScrollTask task, Logger logger, ParentTaskAssigningClient client, - ThreadPool threadPool, DeleteByQueryRequest request, ScriptService scriptService, - ClusterState clusterState, ActionListener listener) { + ThreadPool threadPool, TransportDeleteByQueryAction action, DeleteByQueryRequest request, + ScriptService scriptService, ClusterState clusterState, ActionListener listener) { super(task, // not all nodes support sequence number powered optimistic concurrency control, we fall back to version clusterState.nodes().getMinNodeVersion().onOrAfter(Version.V_6_7_0) == false, // all nodes support sequence number powered optimistic concurrency control and we can use it clusterState.nodes().getMinNodeVersion().onOrAfter(Version.V_6_7_0), - logger, client, threadPool, request, scriptService, listener); + logger, client, threadPool, action, request, listener); useSeqNoForCAS = clusterState.nodes().getMinNodeVersion().onOrAfter(Version.V_6_7_0); } - @Override protected boolean accept(ScrollableHitSource.Hit doc) { // Delete-by-query does not require the source to delete a document diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java index d601f5c06e728..52fc321286a9e 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java @@ -21,21 +21,32 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; +import org.elasticsearch.script.ScriptService; import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.ResourceWatcherService; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.function.Supplier; @@ -69,8 +80,19 @@ public List getRestHandlers(Settings settings, RestController restC new RestRethrottleAction(settings, restController, nodesInCluster)); } + @Override + public Collection createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, ScriptService scriptService, + NamedXContentRegistry xContentRegistry, Environment environment, + NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry) { + return Collections.singletonList(new ReindexSslConfig(environment.settings(), environment, resourceWatcherService)); + } + @Override public List> getSettings() { - return singletonList(TransportReindexAction.REMOTE_CLUSTER_WHITELIST); + final List> settings = new ArrayList<>(); + settings.add(TransportReindexAction.REMOTE_CLUSTER_WHITELIST); + settings.addAll(ReindexSslConfig.getSettings()); + return settings; } } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexSslConfig.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexSslConfig.java new file mode 100644 index 0000000000000..10be40acbc792 --- /dev/null +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexSslConfig.java @@ -0,0 +1,161 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.reindex; + +import org.apache.http.conn.ssl.DefaultHostnameVerifier; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.SecureSetting; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.ssl.SslConfiguration; +import org.elasticsearch.common.ssl.SslConfigurationKeys; +import org.elasticsearch.common.ssl.SslConfigurationLoader; +import org.elasticsearch.env.Environment; +import org.elasticsearch.watcher.FileChangesListener; +import org.elasticsearch.watcher.FileWatcher; +import org.elasticsearch.watcher.ResourceWatcherService; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLContext; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +import static org.elasticsearch.common.settings.Setting.listSetting; +import static org.elasticsearch.common.settings.Setting.simpleString; + +/** + * Loads "reindex.ssl.*" configuration from Settings, and makes the applicable configuration (trust manager / key manager / hostname + * verification / cipher-suites) available for reindex-from-remote. + */ +class ReindexSslConfig { + + private static final Map> SETTINGS = new HashMap<>(); + private static final Map> SECURE_SETTINGS = new HashMap<>(); + + static { + Setting.Property[] defaultProperties = new Setting.Property[] { Setting.Property.NodeScope, Setting.Property.Filtered }; + Setting.Property[] deprecatedProperties = new Setting.Property[] { Setting.Property.Deprecated, Setting.Property.NodeScope, + Setting.Property.Filtered }; + for (String key : SslConfigurationKeys.getStringKeys()) { + String settingName = "reindex.ssl." + key; + final Setting.Property[] properties = SslConfigurationKeys.isDeprecated(key) ? deprecatedProperties : defaultProperties; + SETTINGS.put(settingName, simpleString(settingName, properties)); + } + for (String key : SslConfigurationKeys.getListKeys()) { + String settingName = "reindex.ssl." + key; + final Setting.Property[] properties = SslConfigurationKeys.isDeprecated(key) ? deprecatedProperties : defaultProperties; + SETTINGS.put(settingName, listSetting(settingName, Collections.emptyList(), Function.identity(), properties)); + } + for (String key : SslConfigurationKeys.getSecureStringKeys()) { + String settingName = "reindex.ssl." + key; + SECURE_SETTINGS.put(settingName, SecureSetting.secureString(settingName, null)); + } + } + + private final SslConfiguration configuration; + private volatile SSLContext context; + + public static List> getSettings() { + List> settings = new ArrayList<>(); + settings.addAll(SETTINGS.values()); + settings.addAll(SECURE_SETTINGS.values()); + return settings; + } + + ReindexSslConfig(Settings settings, Environment environment, ResourceWatcherService resourceWatcher) { + final SslConfigurationLoader loader = new SslConfigurationLoader("reindex.ssl.") { + + @Override + protected String getSettingAsString(String key) { + return settings.get(key); + } + + @Override + protected char[] getSecureSetting(String key) { + final Setting setting = SECURE_SETTINGS.get(key); + if (setting == null) { + throw new IllegalArgumentException("The secure setting [" + key + "] is not registered"); + } + return setting.get(settings).getChars(); + } + + @Override + protected List getSettingAsList(String key) throws Exception { + return settings.getAsList(key); + } + }; + configuration = loader.load(environment.configFile()); + reload(); + + final FileChangesListener listener = new FileChangesListener() { + @Override + public void onFileCreated(Path file) { + onFileChanged(file); + } + + @Override + public void onFileDeleted(Path file) { + onFileChanged(file); + } + + @Override + public void onFileChanged(Path file) { + ReindexSslConfig.this.reload(); + } + }; + for (Path file : configuration.getDependentFiles()) { + try { + final FileWatcher watcher = new FileWatcher(file); + watcher.addListener(listener); + resourceWatcher.add(watcher, ResourceWatcherService.Frequency.HIGH); + } catch (IOException e) { + throw new UncheckedIOException("cannot watch file [" + file + "]", e); + } + } + } + + private void reload() { + this.context = configuration.createSslContext(); + } + + /** + * Encapsulate the loaded SSL configuration as a HTTP-client {@link SSLIOSessionStrategy}. + * The returned strategy is immutable, but successive calls will return different objects that may have different + * configurations if the underlying key/certificate files are modified. + */ + SSLIOSessionStrategy getStrategy() { + final HostnameVerifier hostnameVerifier = configuration.getVerificationMode().isHostnameVerificationEnabled() + ? new DefaultHostnameVerifier() + : new NoopHostnameVerifier(); + final String[] protocols = configuration.getSupportedProtocols().toArray(Strings.EMPTY_ARRAY); + final String[] cipherSuites = configuration.getCipherSuites().toArray(Strings.EMPTY_ARRAY); + return new SSLIOSessionStrategy(context, protocols, cipherSuites, hostnameVerifier); + } +} diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java index 9bda99b6e3943..08538b335535d 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java @@ -61,7 +61,7 @@ public void doExecute(Task task, DeleteByQueryRequest request, ActionListener)ReindexRequest::new); this.threadPool = threadPool; this.clusterService = clusterService; @@ -113,6 +115,7 @@ public TransportReindexAction(Settings settings, ThreadPool threadPool, ActionFi this.client = client; remoteWhitelist = buildRemoteWhitelist(REMOTE_CLUSTER_WHITELIST.get(settings)); this.indexNameExpressionResolver = indexNameExpressionResolver; + this.sslConfig = sslConfig; } @Override @@ -129,7 +132,7 @@ protected void doExecute(Task task, ReindexRequest request, ActionListener { ParentTaskAssigningClient assigningClient = new ParentTaskAssigningClient(client, clusterService.localNode(), bulkByScrollTask); - new AsyncIndexBySearchAction(bulkByScrollTask, logger, assigningClient, threadPool, request, scriptService, state, + new AsyncIndexBySearchAction(bulkByScrollTask, logger, assigningClient, threadPool, this, request, state, listener).start(); } ); @@ -197,10 +200,11 @@ static void validateAgainstAliases(SearchRequest source, IndexRequest destinatio /** * Build the {@link RestClient} used for reindexing from remote clusters. * @param remoteInfo connection information for the remote cluster + * @param sslConfig configuration for potential outgoing HTTPS connections * @param taskId the id of the current task. This is added to the thread name for easier tracking * @param threadCollector a list in which we collect all the threads created by the client */ - static RestClient buildRestClient(RemoteInfo remoteInfo, long taskId, List threadCollector) { + static RestClient buildRestClient(RemoteInfo remoteInfo, ReindexSslConfig sslConfig, long taskId, List threadCollector) { Header[] clientHeaders = new Header[remoteInfo.getHeaders().size()]; int i = 0; for (Map.Entry header : remoteInfo.getHeaders().entrySet()) { @@ -233,6 +237,7 @@ static RestClient buildRestClient(RemoteInfo remoteInfo, long taskId, List { + static class AsyncIndexBySearchAction extends AbstractAsyncBulkByScrollAction { /** * List of threads created by this process. Usually actions don't create threads in Elasticsearch. Instead they use the builtin * {@link ThreadPool}s. But reindex-from-remote uses Elasticsearch's {@link RestClient} which doesn't use the @@ -257,7 +262,7 @@ static class AsyncIndexBySearchAction extends AbstractAsyncBulkByScrollAction createdThreads = emptyList(); AsyncIndexBySearchAction(BulkByScrollTask task, Logger logger, ParentTaskAssigningClient client, - ThreadPool threadPool, ReindexRequest request, ScriptService scriptService, ClusterState clusterState, + ThreadPool threadPool, TransportReindexAction action, ReindexRequest request, ClusterState clusterState, ActionListener listener) { super(task, /* @@ -265,7 +270,7 @@ static class AsyncIndexBySearchAction extends AbstractAsyncBulkByScrollAction()); - RestClient restClient = buildRestClient(remoteInfo, task.getId(), createdThreads); + RestClient restClient = buildRestClient(remoteInfo, mainAction.sslConfig, task.getId(), createdThreads); return new RemoteScrollableHitSource(logger, backoffPolicy, threadPool, worker::countSearchRetry, this::finishHim, restClient, remoteInfo.getQuery(), mainRequest.getSearchRequest()); } @@ -296,7 +301,7 @@ protected void finishHim(Exception failure, List indexingFailures, List public BiFunction, ScrollableHitSource.Hit, RequestWrapper> buildScriptApplier() { Script script = mainRequest.getScript(); if (script != null) { - return new ReindexScriptApplier(worker, scriptService, script, script.getParams()); + return new ReindexScriptApplier(worker, mainAction.scriptService, script, script.getParams()); } return super.buildScriptApplier(); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java index 9ed7744f8a27a..b88696d65526c 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java @@ -72,7 +72,7 @@ protected void doExecute(Task task, UpdateByQueryRequest request, ActionListener ClusterState state = clusterService.state(); ParentTaskAssigningClient assigningClient = new ParentTaskAssigningClient(client, clusterService.localNode(), bulkByScrollTask); - new AsyncIndexBySearchAction(bulkByScrollTask, logger, assigningClient, threadPool, request, scriptService, state, + new AsyncIndexBySearchAction(bulkByScrollTask, logger, assigningClient, threadPool, this, request, state, listener).start(); } ); @@ -81,19 +81,19 @@ protected void doExecute(Task task, UpdateByQueryRequest request, ActionListener /** * Simple implementation of update-by-query using scrolling and bulk. */ - static class AsyncIndexBySearchAction extends AbstractAsyncBulkByScrollAction { + static class AsyncIndexBySearchAction extends AbstractAsyncBulkByScrollAction { private final boolean useSeqNoForCAS; AsyncIndexBySearchAction(BulkByScrollTask task, Logger logger, ParentTaskAssigningClient client, - ThreadPool threadPool, UpdateByQueryRequest request, ScriptService scriptService, ClusterState clusterState, + ThreadPool threadPool, TransportUpdateByQueryAction action, UpdateByQueryRequest request, ClusterState clusterState, ActionListener listener) { super(task, // not all nodes support sequence number powered optimistic concurrency control, we fall back to version clusterState.nodes().getMinNodeVersion().onOrAfter(Version.V_6_7_0) == false, // all nodes support sequence number powered optimistic concurrency control and we can use it clusterState.nodes().getMinNodeVersion().onOrAfter(Version.V_6_7_0), - logger, client, threadPool, request, scriptService, listener); + logger, client, threadPool, action, request, listener); useSeqNoForCAS = clusterState.nodes().getMinNodeVersion().onOrAfter(Version.V_6_7_0); } @@ -101,7 +101,7 @@ static class AsyncIndexBySearchAction extends AbstractAsyncBulkByScrollAction, ScrollableHitSource.Hit, RequestWrapper> buildScriptApplier() { Script script = mainRequest.getScript(); if (script != null) { - return new UpdateByQueryScriptApplier(worker, scriptService, script, script.getParams()); + return new UpdateByQueryScriptApplier(worker, mainAction.scriptService, script, script.getParams()); } return super.buildScriptApplier(); } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionMetadataTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionMetadataTestCase.java index 34da9f56b482c..8d67a3bd6760d 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionMetadataTestCase.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionMetadataTestCase.java @@ -28,5 +28,5 @@ protected ScrollableHitSource.BasicHit doc() { return new ScrollableHitSource.BasicHit("index", "type", "id", 0); } - protected abstract AbstractAsyncBulkByScrollAction action(); + protected abstract AbstractAsyncBulkByScrollAction action(); } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java index c9eba1927ef15..76bf2a9160f51 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java @@ -62,7 +62,7 @@ public void execute() { } }; when(scriptService.compile(any(), eq(UpdateScript.CONTEXT))).thenReturn(factory); - AbstractAsyncBulkByScrollAction action = action(scriptService, request().setScript(mockScript(""))); + AbstractAsyncBulkByScrollAction action = action(scriptService, request().setScript(mockScript(""))); RequestWrapper result = action.buildScriptApplier().apply(AbstractAsyncBulkByScrollAction.wrap(index), doc); return (result != null) ? (T) result.self() : null; } @@ -109,5 +109,5 @@ public void testSetOpTypeUnknown() throws Exception { assertThat(e.getMessage(), equalTo("Operation type [unknown] not allowed, only [noop, index, delete] are allowed")); } - protected abstract AbstractAsyncBulkByScrollAction action(ScriptService scriptService, Request request); + protected abstract AbstractAsyncBulkByScrollAction action(ScriptService scriptService, Request request); } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java index cd23bf03add59..3ee651cf96184 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java @@ -48,7 +48,9 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; @@ -71,6 +73,7 @@ import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.internal.InternalSearchResponse; +import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.test.ESTestCase; @@ -675,10 +678,11 @@ private void simulateScrollResponse(DummyAsyncBulkByScrollAction action, TimeVal action.onScrollResponse(lastBatchTime, lastBatchSize, response); } - private class DummyAsyncBulkByScrollAction extends AbstractAsyncBulkByScrollAction { + private class DummyAsyncBulkByScrollAction + extends AbstractAsyncBulkByScrollAction { DummyAsyncBulkByScrollAction() { super(testTask, randomBoolean(), randomBoolean(), AsyncBulkByScrollActionTests.this.logger, - new ParentTaskAssigningClient(client, localNode, testTask), client.threadPool(), testRequest, null, listener); + new ParentTaskAssigningClient(client, localNode, testTask), client.threadPool(), null, testRequest, listener); } @Override @@ -698,6 +702,20 @@ BackoffPolicy buildBackoffPolicy() { } } + private static class DummyTransportAsyncBulkByScrollAction + extends TransportAction { + + + protected DummyTransportAsyncBulkByScrollAction(String actionName, ActionFilters actionFilters, TaskManager taskManager) { + super(actionName, actionFilters, taskManager); + } + + @Override + protected void doExecute(Task task, DummyAbstractBulkByScrollRequest request, ActionListener listener) { + // no-op + } + } + private static class DummyAbstractBulkByScrollRequest extends AbstractBulkByScrollRequest { DummyAbstractBulkByScrollRequest(SearchRequest searchRequest) { super(searchRequest, true); diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteBuildRestClientTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteBuildRestClientTests.java index db32e4813b316..b0b35fbaac74e 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteBuildRestClientTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteBuildRestClientTests.java @@ -22,6 +22,10 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilderTestCase; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.watcher.ResourceWatcherService; import java.util.ArrayList; import java.util.HashMap; @@ -31,6 +35,7 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.synchronizedList; import static org.hamcrest.Matchers.hasSize; +import static org.mockito.Mockito.mock; public class ReindexFromRemoteBuildRestClientTests extends RestClientBuilderTestCase { public void testBuildRestClient() throws Exception { @@ -39,7 +44,7 @@ public void testBuildRestClient() throws Exception { RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT); long taskId = randomLong(); List threads = synchronizedList(new ArrayList<>()); - RestClient client = TransportReindexAction.buildRestClient(remoteInfo, taskId, threads); + RestClient client = TransportReindexAction.buildRestClient(remoteInfo, sslConfig(), taskId, threads); try { assertBusy(() -> assertThat(threads, hasSize(2))); int i = 0; @@ -63,11 +68,18 @@ public void testHeaders() throws Exception { headers, RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT); long taskId = randomLong(); List threads = synchronizedList(new ArrayList<>()); - RestClient client = TransportReindexAction.buildRestClient(remoteInfo, taskId, threads); + RestClient client = TransportReindexAction.buildRestClient(remoteInfo, sslConfig(), taskId, threads); try { assertHeaders(client, headers); } finally { client.close(); } } + + private ReindexSslConfig sslConfig() { + final Environment environment = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build()); + final ResourceWatcherService resourceWatcher = mock(ResourceWatcherService.class); + return new ReindexSslConfig(environment.settings(), environment, resourceWatcher); + } + } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexMetadataTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexMetadataTests.java index 2fe6dd91cd19c..991300e55271f 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexMetadataTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexMetadataTests.java @@ -76,8 +76,8 @@ protected ReindexRequest request() { private class TestAction extends TransportReindexAction.AsyncIndexBySearchAction { TestAction() { - super(ReindexMetadataTests.this.task, ReindexMetadataTests.this.logger, null, ReindexMetadataTests.this.threadPool, request(), - null, null, listener()); + super(ReindexMetadataTests.this.task, ReindexMetadataTests.this.logger, null, ReindexMetadataTests.this.threadPool, + null, request(), null, listener()); } public ReindexRequest mainRequest() { diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexRestClientSslTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexRestClientSslTests.java new file mode 100644 index 0000000000000..f71d124986699 --- /dev/null +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexRestClientSslTests.java @@ -0,0 +1,213 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.reindex; + +import com.sun.net.httpserver.HttpsConfigurator; +import com.sun.net.httpserver.HttpsExchange; +import com.sun.net.httpserver.HttpsParameters; +import com.sun.net.httpserver.HttpsServer; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.ssl.PemKeyConfig; +import org.elasticsearch.common.ssl.PemTrustConfig; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.mocksocket.MockHttpServer; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.watcher.ResourceWatcherService; +import org.hamcrest.Matchers; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import javax.net.ssl.KeyManager; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLHandshakeException; +import javax.net.ssl.SSLPeerUnverifiedException; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509ExtendedKeyManager; +import javax.net.ssl.X509ExtendedTrustManager; +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.nio.file.Path; +import java.security.cert.Certificate; +import java.security.cert.X509Certificate; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; + +import static org.mockito.Mockito.mock; + +/** + * Because core ES doesn't have SSL available, this test uses a mock webserver + * as the remote endpoint. + * This makes it hard to test actual reindex functionality, but does allow us to test that the correct connections are made with the + * right SSL keys + trust settings. + */ +@SuppressForbidden(reason = "use http server") +public class ReindexRestClientSslTests extends ESTestCase { + + private static HttpsServer server; + private static Consumer handler = ignore -> { + }; + + @BeforeClass + public static void setupHttpServer() throws Exception { + InetSocketAddress address = new InetSocketAddress(InetAddress.getLoopbackAddress().getHostAddress(), 0); + SSLContext sslContext = buildServerSslContext(); + server = MockHttpServer.createHttps(address, 0); + server.setHttpsConfigurator(new ClientAuthHttpsConfigurator(sslContext)); + server.start(); + server.createContext("/", http -> { + assert http instanceof HttpsExchange; + HttpsExchange https = (HttpsExchange) http; + handler.accept(https); + // Always respond with 200 + // * If the reindex sees the 200, it means the SSL connection was established correctly. + // * We can check client certs in the handler. + https.sendResponseHeaders(200, 0); + https.close(); + }); + } + + @AfterClass + public static void shutdownHttpServer() { + server.stop(0); + server = null; + handler = null; + } + + private static SSLContext buildServerSslContext() throws Exception { + final SSLContext sslContext = SSLContext.getInstance("TLSv1.2"); + final char[] password = "http-password".toCharArray(); + + final Path cert = PathUtils.get(ReindexRestClientSslTests.class.getResource("http/http.crt").toURI()); + final Path key = PathUtils.get(ReindexRestClientSslTests.class.getResource("http/http.key").toURI()); + final X509ExtendedKeyManager keyManager = new PemKeyConfig(cert, key, password).createKeyManager(); + + final Path ca = PathUtils.get(ReindexRestClientSslTests.class.getResource("ca.pem").toURI()); + final X509ExtendedTrustManager trustManager = new PemTrustConfig(Collections.singletonList(ca)).createTrustManager(); + + sslContext.init(new KeyManager[] { keyManager }, new TrustManager[] { trustManager }, null); + return sslContext; + } + + public void testClientFailsWithUntrustedCertificate() throws IOException { + final List threads = new ArrayList<>(); + final Settings settings = Settings.builder() + .put("path.home", createTempDir()) + .build(); + final Environment environment = TestEnvironment.newEnvironment(settings); + final ReindexSslConfig ssl = new ReindexSslConfig(settings, environment, mock(ResourceWatcherService.class)); + try (RestClient client = TransportReindexAction.buildRestClient(getRemoteInfo(), ssl, 1L, threads)) { + expectThrows(SSLHandshakeException.class, () -> client.performRequest(new Request("GET", "/"))); + } + } + + public void testClientSucceedsWithCertificateAuthorities() throws IOException { + final List threads = new ArrayList<>(); + final Path ca = getDataPath("ca.pem"); + final Settings settings = Settings.builder() + .put("path.home", createTempDir()) + .putList("reindex.ssl.certificate_authorities", ca.toString()) + .build(); + final Environment environment = TestEnvironment.newEnvironment(settings); + final ReindexSslConfig ssl = new ReindexSslConfig(settings, environment, mock(ResourceWatcherService.class)); + try (RestClient client = TransportReindexAction.buildRestClient(getRemoteInfo(), ssl, 1L, threads)) { + final Response response = client.performRequest(new Request("GET", "/")); + assertThat(response.getStatusLine().getStatusCode(), Matchers.is(200)); + } + } + + public void testClientSucceedsWithVerificationDisabled() throws IOException { + assertFalse("Cannot disable verification in FIPS JVM", inFipsJvm()); + final List threads = new ArrayList<>(); + final Settings settings = Settings.builder() + .put("path.home", createTempDir()) + .put("reindex.ssl.verification_mode", "NONE") + .build(); + final Environment environment = TestEnvironment.newEnvironment(settings); + final ReindexSslConfig ssl = new ReindexSslConfig(settings, environment, mock(ResourceWatcherService.class)); + try (RestClient client = TransportReindexAction.buildRestClient(getRemoteInfo(), ssl, 1L, threads)) { + final Response response = client.performRequest(new Request("GET", "/")); + assertThat(response.getStatusLine().getStatusCode(), Matchers.is(200)); + } + } + + public void testClientPassesClientCertificate() throws IOException { + final List threads = new ArrayList<>(); + final Path ca = getDataPath("ca.pem"); + final Path cert = getDataPath("client/client.crt"); + final Path key = getDataPath("client/client.key"); + final Settings settings = Settings.builder() + .put("path.home", createTempDir()) + .putList("reindex.ssl.certificate_authorities", ca.toString()) + .put("reindex.ssl.certificate", cert) + .put("reindex.ssl.key", key) + .put("reindex.ssl.key_passphrase", "client-password") + .build(); + AtomicReference clientCertificates = new AtomicReference<>(); + handler = https -> { + try { + clientCertificates.set(https.getSSLSession().getPeerCertificates()); + } catch (SSLPeerUnverifiedException e) { + logger.warn("Client did not provide certificates", e); + clientCertificates.set(null); + } + }; + final Environment environment = TestEnvironment.newEnvironment(settings); + final ReindexSslConfig ssl = new ReindexSslConfig(settings, environment, mock(ResourceWatcherService.class)); + try (RestClient client = TransportReindexAction.buildRestClient(getRemoteInfo(), ssl, 1L, threads)) { + final Response response = client.performRequest(new Request("GET", "/")); + assertThat(response.getStatusLine().getStatusCode(), Matchers.is(200)); + final Certificate[] certs = clientCertificates.get(); + assertThat(certs, Matchers.notNullValue()); + assertThat(certs, Matchers.arrayWithSize(1)); + assertThat(certs[0], Matchers.instanceOf(X509Certificate.class)); + final X509Certificate clientCert = (X509Certificate) certs[0]; + assertThat(clientCert.getSubjectDN().getName(), Matchers.is("CN=client")); + assertThat(clientCert.getIssuerDN().getName(), Matchers.is("CN=Elastic Certificate Tool Autogenerated CA")); + } + } + + private RemoteInfo getRemoteInfo() { + return new RemoteInfo("https", server.getAddress().getHostName(), server.getAddress().getPort(), "/", new BytesArray("test"), + "user", "password", Collections.emptyMap(), RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT); + } + + @SuppressForbidden(reason = "use http server") + private static class ClientAuthHttpsConfigurator extends HttpsConfigurator { + ClientAuthHttpsConfigurator(SSLContext sslContext) { + super(sslContext); + } + + @Override + public void configure(HttpsParameters params) { + params.setWantClientAuth(true); + } + } +} diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexScriptTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexScriptTests.java index 732bc9acdb662..b7fa35f7877f3 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexScriptTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexScriptTests.java @@ -20,9 +20,14 @@ package org.elasticsearch.index.reindex; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.common.lucene.uid.Versions; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.transport.TransportService; +import org.mockito.Mockito; +import java.util.Collections; import java.util.Map; import static org.hamcrest.Matchers.containsString; @@ -103,7 +108,11 @@ protected ReindexRequest request() { @Override protected TransportReindexAction.AsyncIndexBySearchAction action(ScriptService scriptService, ReindexRequest request) { - return new TransportReindexAction.AsyncIndexBySearchAction(task, logger, null, threadPool, request, scriptService, - null, listener()); + TransportService transportService = Mockito.mock(TransportService.class); + ReindexSslConfig sslConfig = Mockito.mock(ReindexSslConfig.class); + TransportReindexAction transportAction = new TransportReindexAction(Settings.EMPTY, threadPool, + new ActionFilters(Collections.emptySet()), null, null, scriptService, null, null, transportService, sslConfig); + return new TransportReindexAction.AsyncIndexBySearchAction(task, logger, null, threadPool, transportAction, request, + null, listener()); } } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryMetadataTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryMetadataTests.java index 95ee787f13f63..6d6f034827ab8 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryMetadataTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryMetadataTests.java @@ -45,7 +45,7 @@ protected UpdateByQueryRequest request() { private class TestAction extends TransportUpdateByQueryAction.AsyncIndexBySearchAction { TestAction() { super(UpdateByQueryMetadataTests.this.task, UpdateByQueryMetadataTests.this.logger, null, - UpdateByQueryMetadataTests.this.threadPool, request(), null, ClusterState.EMPTY_STATE, listener()); + UpdateByQueryMetadataTests.this.threadPool, null, request(), ClusterState.EMPTY_STATE, listener()); } @Override diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java index 0eb2a1cfb7d0a..22d4ac0eab799 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java @@ -19,13 +19,17 @@ package org.elasticsearch.index.reindex; +import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.transport.TransportService; +import java.util.Collections; import java.util.Date; import java.util.Map; import static org.hamcrest.Matchers.containsString; +import static org.mockito.Mockito.mock; public class UpdateByQueryWithScriptTests extends AbstractAsyncBulkByScrollActionScriptTestCase { @@ -54,7 +58,10 @@ protected UpdateByQueryRequest request() { @Override protected TransportUpdateByQueryAction.AsyncIndexBySearchAction action(ScriptService scriptService, UpdateByQueryRequest request) { - return new TransportUpdateByQueryAction.AsyncIndexBySearchAction(task, logger, null, threadPool, request, scriptService, + TransportService transportService = mock(TransportService.class); + TransportUpdateByQueryAction transportAction = new TransportUpdateByQueryAction(threadPool, + new ActionFilters(Collections.emptySet()), null, transportService, scriptService, null); + return new TransportUpdateByQueryAction.AsyncIndexBySearchAction(task, logger, null, threadPool, transportAction, request, ClusterState.EMPTY_STATE, listener()); } } diff --git a/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/README.txt b/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/README.txt new file mode 100644 index 0000000000000..4221d479442d9 --- /dev/null +++ b/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/README.txt @@ -0,0 +1,25 @@ +# ca.p12 + +$ES_HOME/bin/elasticsearch-certutil ca --out ca.p12 --pass "ca-password" --days 9999 + +# ca.pem + +openssl pkcs12 -info -in ./ca.p12 -nokeys -out ca.pem -passin "pass:ca-password" + +# http.p12 + +$ES_HOME/bin/elasticsearch-certutil cert --out http.zip --pass "http-password" \ + --days 9999 --pem --name "http" \ + --ca ca.p12 --ca-pass "ca-password" \ + --dns=localhost --dns=localhost.localdomain --dns=localhost4 --dns=localhost4.localdomain4 --dns=localhost6 --dns=localhost6.localdomain6 \ + --ip=127.0.0.1 --ip=0:0:0:0:0:0:0:1 +unzip http.zip +rm http.zip + +# client.p12 + +$ES_HOME/bin/elasticsearch-certutil cert --out client.zip --pass "client-password" \ + --name "client" --days 9999 --pem \ + --ca ca.p12 --ca-pass "ca-password" +unzip client.zip +rm client.zip diff --git a/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/ca.p12 b/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/ca.p12 new file mode 100644 index 0000000000000000000000000000000000000000..332a75aec64c19fcb8a944b1b1300d353748b32f GIT binary patch literal 2527 zcmY+EcRUmhAIFb7936hZNI^qE#Dl{?cyP!C z4nW|kEdNna8G`UsdKXyxA{x~6|NCN~qo5?>LB|L@=l}twru#p>80P`8_O{SBju(+4 zWR0-gU3sZ#?GwmNN+6*99taQW4o}c&J974UKW}p9c-loNE#IE0juZ5|rb=nW zKu6{GAeJc1Fp2 z3?V2ObMj+e-Ei)v%gS!$7KV1+9~!)b*8#5=(dTW+sWBoFLEyla)2x~8+5wFDQ(SMr zH{yw=cMbjW(f8TA%7@*ux`PVbY>TWcf&1ff=6UW44eQobVRR%1&eGi@Vl}1ul7Cc( zss1k$Rgza^Nw?Vmf6IAv?1$JgiZ%k@t5(-ew3|c6awSGYX?K%O26Pg;!D`yCOWVmY zFXU!L;y-VqCzrH5TiMCfr5SC+9wgmratnzx5IV`!@_iUar#g=SOCn>bM6Tt>^49nS z@l8SJ3z;P1J$L@#Mo}Ec+<(MRsp`wo>z&IS@^{P+iRj|+v$Z;5a?7GE~yffxLT3$Onr;wc!hUg^%>Azs`HD84PmU(4}B+pM0)@F zogjg^o~AmrYF%!NYfd|X9WEpU=k^*@dgNxs*qKHFs!+%z?vkTua8j$8)W;&H5j&ID zHlbC|l^JlR8t|_BozyJtzje4O+9^kc89%Ox1b@bJD6gN8d9~FwwTDk<(kmkdM1m$P zP%WcR+7@A-kiFB2zmjhqNF~jY<1&TG_J2yh(z#S0=g=JB7t*?kwyGs^5d}zKpB3a? zME~l0Y0x71D6R6FHDts?|16V0@y+O7Hr(Ei-ZRqIW|C_#Lc9^J)Nl6EXv;TQ^XRhR zQr!5D;`3bBLyy-^ardL(;;A*lEW!lhxa1JhPAH_=OzSL5I{CcnoROlq{K<)6sS0jeDrisbpJ3gO6bt2oCN4_z`T zT0<`XB$R{(wlW50*|56x3jYg8J18klT8tn+q1c0kR+3_NKjf`e)yb48E1q9P))@mwoiW2vMm z&tTj?pTKrH6l*9RNR?0mc&=8H?0tl3TM27G`R3OoMD+mpuex@s#rD}>Mam+s{H01T zE0UTQ1h58JBKZG?Y)DoR%Y8>@TL@C>mJC8xT0%k&iIBOFChK2EG}IR!rMf^AloS92 z9(eS3L-7x@Qvb)SX`Ry;FmIBe?jCA3;@#-#9`dc(f6W?<2e#Dv+7wFH7*5py{gX_N zymE&7(LVya({r_A1J@Qvb+W2g>#nEGAI0#r$3io?e>8db^4~D_kA2d|QZj^XoC@f5 z@EN}SguN->2A#ct8x%hBmhXq8t50X7{5gbqORt)mE8H&?)qytQ!bm?oW%A8`AJRf` zFLL7yQc47=G&zjJU1y@4&C4*U@{pT|3hj8&FA`lQ9vFJ>SwYM8T9kXzl%j}7@Jsg4 zGuxT$MhtY|id-s?Xn=S)`9b9cixn1Cmyx$X!{xNHO#P)Re5>RJY{}|A9}VbxTNx8q^&YZhf=)AEhuv^7U73YU>S@hei5)y+ga330=D&V;$U$ z8W9DfF3v&tgojw73(U5rrKtK(@ON~pYRcfWK{S!I_F%@j9m@7p8I-l@Up}NFztZm8 z%6xuwbJ1&>Z2bt6)oBD#2yH&)do(;M1K;!v36Q?LSo5-^6wGA)$x3jh(kw*$ z_U~{$mFeE~Jiml`I#MG8w*~(;<#=ihwwBb;F*K-N&G}NzB}&hL{<%CTETxzae=`)HZqaJ6SS##!`@ZqC)GfcKTkrJ;7L_q$cut z;WrrZlj2O4cuSng`{ON{h~6pF$|hChIju&y(d^Zy%Zy%&TjZ(`2t1CLVp#NT1)q@D z2E?HRM4$0;-90O;q(59;NS3N%Lco#M=))k)CJ z{LmPzQ>4&O`keR_&v}*7+GD51;7f${^=#XCeP(~#H{4nf;Ix`&*;)z1wzGua0M=pT z^sVA&wcl|s(aegGC_)RkH1!dz>fE}-G8aB85^r2x*P1x_#z^SO&rGQh?@+iVF^y*n z4QCYE5A{xMl_agR+jBgYsZMFTMwkxtxe-0DC-_{&ncn27Ue44m3A(jQ`$%?qre)_< zvu8_=2$5td{et2vGV@@7w86G2>$d{IrRWrP>oHv8{!%lz`Q}B6z{bgBuHmM7OtD7m zSmfQLN;^)oCRcJ)`Bp Z*!*5kK%{lsCg6zGH{GI*5=aqF{5P0cyMq7# literal 0 HcmV?d00001 diff --git a/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/ca.pem b/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/ca.pem new file mode 100644 index 0000000000000..ee758ca3e6370 --- /dev/null +++ b/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/ca.pem @@ -0,0 +1,25 @@ +Bag Attributes + friendlyName: ca + localKeyID: 54 69 6D 65 20 31 35 34 37 30 38 36 32 32 39 31 30 37 +subject=/CN=Elastic Certificate Tool Autogenerated CA +issuer=/CN=Elastic Certificate Tool Autogenerated CA +-----BEGIN CERTIFICATE----- +MIIDSTCCAjGgAwIBAgIUacmv5ElKJ1cs9n61tEpy5KM3Dv0wDQYJKoZIhvcNAQEL +BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l +cmF0ZWQgQ0EwHhcNMTkwMTEwMDIxMDI5WhcNNDYwNTI3MDIxMDI5WjA0MTIwMAYD +VQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJ0rA35tPl0FN+BPk2YfmET9 +MvDWFLvfL2Z1aw1q1vnd12K9zumjN6veilHA2Iw/P4LG/mkQZvY4bDPgibRD7hbE +vwPoju4vr614tw60+FlkpO6HezYo2I3cni1//Gehhs5EW2P3g7Lw7UNCOAfcR2QQ +p/dtwXYWzXHY9jTevQSv2q/x5jWKZT4ltaQExzvXAcxRGqyWV6d5vol3KH/GpCSI +SQvRmRVNQGXhxi66MjCglGAM2oicd1qCUDCrljdFD/RQ1UzqIJRTXZQKOno1/Em9 +xR0Cd5KQapqttPusAO6uZblMO2Ru+XjCD6Y0o41eCDbkd0xA3/wgP3MD5n41yncC +AwEAAaNTMFEwHQYDVR0OBBYEFJTry9da5RZbbELYCaWVVFllSm8DMB8GA1UdIwQY +MBaAFJTry9da5RZbbELYCaWVVFllSm8DMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI +hvcNAQELBQADggEBADA6qhC35PwuL7LRddbhjjW8U/cCmG9m7AIvH6N+Mw/k76gt +tJkEDxztMHUG+A2IPyEcYm7MLr1D8xEQYsq0x4pzFcQnMSQDv4WTK35vRxMtaqwA +WZTyA+DibBknbaP1z3gNhR9A0TKx4cPagN3OYFvAi/24abf8qS6D/bcOiPDQ4oPb +DVhmhqt5zduDM+Xsf6d4nsA6sf9+4AzneaZKGAMgCXgo4mYeP7M4nMQk0L3ao9Ts ++Usr8WRxc4xHGyb09fsXWSz7ZmiJ6iXK2NvRUq46WCINLONLzNkx29WEKQpI3wh4 +kyx6wF9lwBF06P1raFIBMeMOCkqDc+nj7A91PEA= +-----END CERTIFICATE----- diff --git a/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/client/client.crt b/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/client/client.crt new file mode 100644 index 0000000000000..337d24e2493ac --- /dev/null +++ b/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/client/client.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDIDCCAgigAwIBAgIUNOREYZadZ2EVkJ1m8Y9jnVmWmtAwDQYJKoZIhvcNAQEL +BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l +cmF0ZWQgQ0EwHhcNMTkwMTEwMDIxMDMyWhcNNDYwNTI3MDIxMDMyWjARMQ8wDQYD +VQQDEwZjbGllbnQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCCP2LE +nws2+ZIwSQ3IvIhVfrueUmNt7Y5TdhhwO32p2wC4ZA62J9L8klAzt7R+izcL/qbF +65inbXM0A7ge/2wZ09kbqBk5uS8jDetJS8lQmWVZDHfVi8g/yDMWklz2mQYleYmU +HPyIplai3P3KBoT8HurzHw2C953EZ2HiANFnGoEPZZ5ytcT2WenxuU5kSXSxuDyn +8/dCVHEQL1Yipr2LQKYQAHotjo56OhyL9KS5YPjzSFREeyRfQinssTmpGFsua/PK +Vqj+hRdkaqRfiqPq3wxn8oOSpZLQe58O1e7OlqgjkPuZdjZ0pQ7KJj7N3fUQNSeg +2VC2tk8zv/C/Qr2bAgMBAAGjTTBLMB0GA1UdDgQWBBQziDNuD83ZLwEt1e1txYJu +oSseEDAfBgNVHSMEGDAWgBSU68vXWuUWW2xC2AmllVRZZUpvAzAJBgNVHRMEAjAA +MA0GCSqGSIb3DQEBCwUAA4IBAQAPpyWyR4w6GvfvPmA1nk1qd7fsQ1AucrYweIJx +dTeXg3Ps1bcgNq9Us9xtsKmsoKD8UhtPN6e8W8MkMmri+MSzlEemE+pJZrjHEudi +Sj0AFVOK6jaE0lerbCnTQZvYH+J9Eb1i9RP7XHRShkR4MWgy2BzlENk9/LRbr84W +Yf5TuM9+ApiiiOoX9UfSGBzNnqwhJNpG9yJ+HnQSqTnJJc/wL0211zLme9I/nhf0 +kQx6mPedJ3gGoJ8gqz38djIrhJDxq+0Bd9SsdlR6yT+1+bY7hinYx2eLV91AybZ4 +x07Kyl174DD41PYaE1AtoLlrMrQ5BG7Md50Am+XXOR1X1dkZ +-----END CERTIFICATE----- diff --git a/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/client/client.key b/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/client/client.key new file mode 100644 index 0000000000000..95e11f79cea24 --- /dev/null +++ b/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/client/client.key @@ -0,0 +1,30 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,81AB10154C04B38F + +0L6Buvpeg6QHh/mbYp/3bXDCsu0k0j5xPdIGWd6NCOdb24OQFsOjeA2WuPqs0WWF +gzVrjh984biS3IqeglEr6X6PfVJ0QOgBkq0XgSBXhuoRJL/302N9oPGsf8T8oW9t +pqR/JIB2L7lMbJlJYSjMl0YQT3hWpo2BlrtSIc/GWOKfjDNWc9BL+oHvKJwql1lb +n4yMvYFYJDqgzgxa3r4IIQNsCn3SP+gqbTx9vF6StOIroV51BdSL4IGWRvqnMJrh +ybk1EHSLR1oGcONLU4Ksi33UxdImG70SsnoH/NnInDvV2bxmxmgf5SfYKtxFhoxz +0hISKTMTerPGtRQ5p8wtEi/ULKyInK+qF3tLgZa+S5VbByjDnUo2dCcbDDSkH5pO +uczJ2bs1kJegpCrUueJdbi9OX2upmF+tJb9+5hzFTvey8dUWTEpdiN0xbp4BLfNd +Yp4sMHZovsDJKIjDb0NbXRgLeFh1ijlLPhKwIXWTF3BaCKcSw34Qv22YPwn3qNuw +0KuUPAo0B65R/hoJguvtks8QAXe0S1jZS/fAlQCoIB0TIduy1qkyje+AnSW+1RL0 +ysBxLqbvRUqWlgnu7/28V4FD8JNu3O+UGBEelXlfokLgCBZ6lSys2d3Zy/XVBnG0 +cPl59if+fxKaMWlhFvMLFBup1Y4a/1zA7Sx6kkhvawekHr40NcG4kLHJ+O6UoM4d +/ibnbfIksLNkuo/nwoEcKp7W6SxafV0hROdxClkGKild66rnHtk4IGATjaBqt9nr +FuO3vRtLuUMS+/4kpvhMwl0RhX2/i6xgV+klWNYNu1JTGDFvdG3qfiY2w88EIbGe +rn8JEvRtaH/XNeGdhBwbuObvTifiHyYzA1i5Zh8zvE2+Dthlk19jbBoOUx//LOi2 +JrNkAsqQCF4HXh7n9HWA/ZrKTP7Xvkig6Vf7M2Y/tO361LSJfzKcRFLpl0P2ntEv +XwFOqTvOURERTVr4sBLOVPRAhIs3yvkI5xfurXzbRWtSeLgrMoDgJlXIQbuXd8sq +zIBLqvYf2bcroB66XJqX1IFWEstym/NHGcbrwjR5Fn2p3YAtXnIbw8VhHwV+LIOl +ky/wH9vbnML/DE81qFqRe8vNZw2sGn9skOyU/QvKeV1NRHYZSV3hMx82bPnjgFeB +ilzkb8FEPOAOJ0m44Q3C9eUoazJT8aCuRIAgSL43se1E2pFlIXQTfYRARaWEkSf9 +0hXqQJc17b+Hj0ire3PUqbG3+/l1qMhhIHwq7Kuyy2neTuW/DXbXp2AMv/bLcnHH +apVeRZaYXVSnGXJNk2CeRnCs8OGir8g5zkH+fmVb9knt6TL2oFIsQqULyrLolhfe +6Q8mLzq/sd+w+VuN1n/5+RQqOJZWEkLFzQPx8wTqeTB19OE0gjncrqzCHq7INqRe +tGClWOj/yL0Sciu3ctVGz1VAbgeBKnLdKm2TX4oFB4OG4E7GMXIL7hGxjtjLAVMW +XNc3ZYNQra+iPqJtFxnmbrF2Sn0Wr0hcAT1V0A0TRKe/n0lpUrfhTy/q4DUlOVKG +qdCsTGoYXObpUWU5G9GyCVWWRJyrTxJcBZ9KWJu9Y/aMFzoa2n0HQw== +-----END RSA PRIVATE KEY----- diff --git a/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/http/http.crt b/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/http/http.crt new file mode 100644 index 0000000000000..309ade87fbd78 --- /dev/null +++ b/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/http/http.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDsjCCApqgAwIBAgIUXxlg/0/g3UYekXWBRpkHM84EYfIwDQYJKoZIhvcNAQEL +BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l +cmF0ZWQgQ0EwHhcNMTkwMTEwMDIxMDMwWhcNNDYwNTI3MDIxMDMwWjAPMQ0wCwYD +VQQDEwRodHRwMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAi8VQaSR6 +uqgT1Rkw+a39OSXcXuhJBVdoO+AyYPK7hdUTxj1aqnXkKeAiNGpe/J+uXZ837Spy +rmBZS3k6S5hLEceF2xug8yrR7RYEZ+JvGlRgg/jj+61gGbHAD314+vvu0YUo06YG +wbz9AnjJA/sMbsCp3iSzWIkwZBZcCoZ/YsG4I89LSjYL3YmRi2193WMX6/OfQYMN +Fkv61r/iwBEkgJ14cUSYe3norGuQfZuXSh5kI5D5R7q7Bmb0um+jzY/l62kj3oR1 +YWo3g6DdU/Bc/3/KmEEVXIfdTonMBMyL8PvYORoMKrYdph3E8e39ZQhPeBJNJKw0 +XzsZFzIUlTw0kQIDAQABo4HgMIHdMB0GA1UdDgQWBBTiqknjZLa5E1BneHRvTkNa +Bm4nNTAfBgNVHSMEGDAWgBSU68vXWuUWW2xC2AmllVRZZUpvAzCBjwYDVR0RBIGH +MIGEgglsb2NhbGhvc3SCF2xvY2FsaG9zdDYubG9jYWxkb21haW42hwR/AAABhxAA +AAAAAAAAAAAAAAAAAAABggpsb2NhbGhvc3Q0ggpsb2NhbGhvc3Q2ghVsb2NhbGhv +c3QubG9jYWxkb21haW6CF2xvY2FsaG9zdDQubG9jYWxkb21haW40MAkGA1UdEwQC +MAAwDQYJKoZIhvcNAQELBQADggEBAIZr8EhhCbNyc6iHzUJ/NrUGht5RDHUKN9WU +2fd+SJlWijQYGoFW6LfabmYxIVPAFtYzUiA378NFoOZZ4kdC3gQng8izvS2UDcO6 +cAG5q/dxop3VXqcLeK3NpH2jd83M8VZaOThPj/F07eTkVX+sGu+7VL5Lc/XPe8JS +HhH2QtcTPGPpzPnWOUMLpRy4mh5sDyeftWr2PTFgMXFD6dtzDvaklGJvr1TmcOVb +BFYyVyXRq6v8YsrRPp0GIl+X3zd3KgwUMuEzRKkJgeI1lZRjmHMIyFcqxlwMaHpv +r1XUmz02ycy6t3n+2kCgfU6HnjbeFh55KzNCEv8TXQFg8Z8YpDA= +-----END CERTIFICATE----- diff --git a/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/http/http.key b/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/http/http.key new file mode 100644 index 0000000000000..8b8d3b4083c67 --- /dev/null +++ b/modules/reindex/src/test/resources/org/elasticsearch/index/reindex/http/http.key @@ -0,0 +1,30 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,127A4142FA81C5A1 + +dP6oSAUl47KCnP0YZSX108qcX5s2nVGpD0qtnVQg89mLVFd7IxpKQaIuODSadRTo +AD0KINITy3ZwUr/TTJgERu88baBsTHv3PLEe7TpQI2DGGDz3aZfO9e6Jvglbdi5b +CBLaxRXGGhhH9YH0E87Lp3JEwg4udWmlNahGIhbqNheZNTtDKt+Lx80TyyIml2r/ +GAhjT4UPvIRrATFAcL/3EKOjRqvb6SeGnZu21n2TSmsBEr02gC0Ox3qmsnRM3kvU +jCuUzWTzJSQLXZwZuMtv5srOSFAbU8EklFXNhWJU/7GBy215aAAW48hCzkPMVEbg +oeH4nuze/Uulih9UxJGCBIpvfTnksyMRGP/zdy1mnKuqQk+yI0n7JWMJL8QoDQc8 +XvzqOmKLdBVezmzOVP/PyMAhYWetILh/1UesjyJot2hwSXPAxqBHPVA9bnmel6CQ +VccNSwaK120yT5YhkUMFc0AmUpztzNMQzJ10g1dW+Qsr+n4vtFmAuTvBgogNNVXn +eX1hbbiXGO1Fw4OMu6qTJ4T/P+VFb0CxoxETWeqdjcs4LGbeqF68nayEsW0ZzhbI +W5c+JAbW18Kb+k/KzKZTtJEXBw6B/2FMe9x9z3BIpVhplM2KsNk7joWnumD8LfUT +ORRHUPV7bkdiDsn2CRaevubDQiChcjsdLWhG7JLm54ttyif7/X7htGOXPZLDLK8B +Vxe09B006f7lM0tXEx8BLFDNroMLlrxB4K5MlwWpS3LLqy4zDbHka2I3s/ST/BD4 +0EURHefiXJkR6bRsfGCl3JDk0EakcUXM+Ob5/2rC/rPXO2pC0ksiQ2DSBm7ak9om +vlC7dIzVipL0LZTd4SUDJyvmK4Ws6V98O5b+79To6oZnVs5CjvcmpSFVePZa5gm/ +DB8LOpW4jklz+ybJtHJRbEIzmpfwpizThto/zLbhPRyvJkagJfWgXI0j+jjKZj+w +sy1V8S44aXJ3GX9p4d/Grnx6WGvEJSV0na7m3YQCPEi5sUgr+EMizGUYstSSUPtU +XhxQRZ95K2cKORul9vzG3zZqqvi73Ju5vu9DLmmlI00sLzyVGFtvkuhrF2p7XclM +GU/rMOeMClMb6qyCzldSs84Anhlh/6mYri6uYPhIGvxqtH44FTbu1APvZp0s2rVm +ueClHG78lat+oqWFpbA8+peT0dMPdSKDAFDiHsGoeWCIoCF44a84bJX35OZk+Y4a ++fDFuSiKYBMfAgqf/ZNzV4+ySka7dWdRQ2TDgIuxnvFV1NgC/ir3/mPgkf0xZU5d +w8T+TW6T8PmJfHnW4nxgHaqgxMoEoPm8zn0HNpRFKwsDYRFfobpCXnoyx50JXxa4 +jg095zlp8X0JwconlGJB1gfeqvS2I50WEDR+2ZtDf7fUEnQ3LYJzP4lSwiSKiQsQ +MPjy0SMQnqmWijylLYKunTl3Uh2DdYg4MOON662H3TxQW8TCYwK2maKujwS9VFLN +GtRGlLrOtrOfHBSwDCujFjqEmQBsF/y2C6XfMoNq6xi5NzREGmNXYrHbLvl2Njwm +WB1ouB4JzmEmb1QNwxkllBAaUp1SJGhW2+fYOe0zjWOP9R4sUq4rRw== +-----END RSA PRIVATE KEY----- From 1a93976ff7385ea1a505aef658e938da310431b7 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 31 Jan 2019 02:45:42 -0500 Subject: [PATCH 083/100] Correct arg names when update mapping/settings from leader (#38063) These two arguments are not named incorrectly and caused confusion. --- .../xpack/ccr/action/ShardFollowNodeTask.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTask.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTask.java index 233085c0a6857..a4f02707bc40f 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTask.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTask.java @@ -133,21 +133,21 @@ void start( } // updates follower mapping, this gets us the leader mapping version and makes sure that leader and follower mapping are identical - updateMapping(0L, followerMappingVersion -> { + updateMapping(0L, leaderMappingVersion -> { synchronized (ShardFollowNodeTask.this) { - currentMappingVersion = followerMappingVersion; + currentMappingVersion = leaderMappingVersion; } - updateSettings(followerSettingsVersion -> { + updateSettings(leaderSettingsVersion -> { synchronized (ShardFollowNodeTask.this) { - currentSettingsVersion = followerSettingsVersion; + currentSettingsVersion = leaderSettingsVersion; } LOGGER.info( "{} following leader shard {}, follower global checkpoint=[{}], mapping version=[{}], settings version=[{}]", params.getFollowShardId(), params.getLeaderShardId(), followerGlobalCheckpoint, - followerMappingVersion, - followerSettingsVersion); + leaderMappingVersion, + leaderSettingsVersion); coordinateReads(); }); }); From 160d1bd4dd59fd669d062e7f016bf14ac6be1fe4 Mon Sep 17 00:00:00 2001 From: Alexander Reelsen Date: Thu, 31 Jan 2019 08:52:35 +0100 Subject: [PATCH 084/100] Work around JDK8 timezone bug in tests (#37968) The timezone GMT0 cannot be properly parsed on java8. The randomZone() method now excludes GMT0, if java8 is used. Closes #37814 --- .../elasticsearch/search/query/SearchQueryIT.java | 5 +---- .../java/org/elasticsearch/test/ESTestCase.java | 13 ++++++++++++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java b/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java index 58302428848b3..ac0152582352d 100644 --- a/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java +++ b/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java @@ -426,14 +426,11 @@ public void testDateRangeInQueryString() { assertThat(e.toString(), containsString("unit [D] not supported for date math")); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37814") // Issue #7880 public void testDateRangeInQueryStringWithTimeZone_7880() { //the mapping needs to be provided upfront otherwise we are not sure how many failures we get back //as with dynamic mappings some shards might be lacking behind and parse a different query - assertAcked(prepareCreate("test").addMapping( - "type", "past", "type=date" - )); + assertAcked(prepareCreate("test").addMapping("type", "past", "type=date")); ZoneId timeZone = randomZone(); String now = DateFormatter.forPattern("strict_date_optional_time").format(Instant.now().atZone(timeZone)); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 61353d42ef178..235cf2ad24fb9 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -48,6 +48,7 @@ import org.apache.lucene.util.TimeUnits; import org.elasticsearch.Version; import org.elasticsearch.bootstrap.BootstrapForTesting; +import org.elasticsearch.bootstrap.JavaVersion; import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -784,7 +785,17 @@ public static TimeZone randomTimeZone() { * generate a random TimeZone from the ones available in java.time */ public static ZoneId randomZone() { - return ZoneId.of(randomFrom(JAVA_ZONE_IDS)); + // work around a JDK bug, where java 8 cannot parse the timezone GMT0 back into a temporal accessor + // see https://bugs.openjdk.java.net/browse/JDK-8138664 + if (JavaVersion.current().getVersion().get(0) == 8) { + ZoneId timeZone; + do { + timeZone = ZoneId.of(randomFrom(JAVA_ZONE_IDS)); + } while (timeZone.equals(ZoneId.of("GMT0"))); + return timeZone; + } else { + return ZoneId.of(randomFrom(JAVA_ZONE_IDS)); + } } /** From b94acb608b58e7918c2ec1296db83dbc17391fe3 Mon Sep 17 00:00:00 2001 From: Alexander Reelsen Date: Thu, 31 Jan 2019 08:55:40 +0100 Subject: [PATCH 085/100] Speed up converting of temporal accessor to zoned date time (#37915) The existing implementation was slow due to exceptions being thrown if an accessor did not have a time zone. This implementation queries for having a timezone, local time and local date and also checks for an instant preventing to throw an exception and thus speeding up the conversion. This removes the existing method and create a new one named DateFormatters.from(TemporalAccessor accessor) to resemble the naming of the java time ones. Before this change an epoch millis parser using the toZonedDateTime method took approximately 50x longer. Relates #37826 --- .../time/DateFormatterFromBenchmark.java | 53 ++++++ .../client/ml/job/util/TimeUtil.java | 2 +- .../ingest/common/DateFormat.java | 34 +++- .../ingest/common/DateFormatTests.java | 20 ++- .../common/time/DateFormatters.java | 162 ++++++++---------- .../common/time/JavaDateMathParser.java | 4 +- .../index/mapper/DateFieldMapper.java | 2 +- .../elasticsearch/common/RoundingTests.java | 2 +- .../joda/JavaJodaTimeDuellingTests.java | 2 +- .../common/time/DateFormattersTests.java | 12 +- .../common/time/JavaDateMathParserTests.java | 4 +- .../index/mapper/DateFieldTypeTests.java | 14 +- .../aggregations/bucket/DateHistogramIT.java | 10 +- .../bucket/DateHistogramOffsetIT.java | 2 +- .../composite/CompositeAggregatorTests.java | 2 +- .../DateHistogramAggregatorTests.java | 2 +- .../bucket/histogram/DateHistogramTests.java | 6 +- .../pipeline/AvgBucketAggregatorTests.java | 2 +- .../CumulativeSumAggregatorTests.java | 2 +- .../pipeline/DateDerivativeIT.java | 24 +-- .../aggregations/pipeline/MovFnUnitTests.java | 2 +- .../org/elasticsearch/license/DateUtils.java | 4 +- 22 files changed, 224 insertions(+), 143 deletions(-) create mode 100644 benchmarks/src/main/java/org/elasticsearch/benchmark/time/DateFormatterFromBenchmark.java diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/time/DateFormatterFromBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/time/DateFormatterFromBenchmark.java new file mode 100644 index 0000000000000..86753dba02b0b --- /dev/null +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/time/DateFormatterFromBenchmark.java @@ -0,0 +1,53 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.benchmark.time; + +import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.common.time.DateFormatters; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import java.time.temporal.TemporalAccessor; +import java.util.concurrent.TimeUnit; + +@Fork(3) +@Warmup(iterations = 10) +@Measurement(iterations = 10) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +@SuppressWarnings("unused") //invoked by benchmarking framework +public class DateFormatterFromBenchmark { + + private final TemporalAccessor accessor = DateFormatter.forPattern("epoch_millis").parse("1234567890"); + + @Benchmark + public TemporalAccessor benchmarkFrom() { + // benchmark an accessor that does not contain a timezone + // this used to throw an exception earlier and thus was very very slow + return DateFormatters.from(accessor); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/util/TimeUtil.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/util/TimeUtil.java index 4c21ffb2175b2..254979a360d0f 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/util/TimeUtil.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/util/TimeUtil.java @@ -39,7 +39,7 @@ public static Date parseTimeField(XContentParser parser, String fieldName) throw if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER) { return new Date(parser.longValue()); } else if (parser.currentToken() == XContentParser.Token.VALUE_STRING) { - return new Date(DateFormatters.toZonedDateTime(DateTimeFormatter.ISO_INSTANT.parse(parser.text())).toInstant().toEpochMilli()); + return new Date(DateFormatters.from(DateTimeFormatter.ISO_INSTANT.parse(parser.text())).toInstant().toEpochMilli()); } throw new IllegalArgumentException( "unexpected token [" + parser.currentToken() + "] for [" + fieldName + "]"); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java index a15bc5049801f..8629f5f1fa321 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java @@ -28,12 +28,23 @@ import java.time.Instant; import java.time.LocalDate; +import java.time.ZoneId; import java.time.ZoneOffset; import java.time.ZonedDateTime; +import java.time.temporal.ChronoField; import java.time.temporal.TemporalAccessor; +import java.util.Arrays; +import java.util.List; import java.util.Locale; import java.util.function.Function; +import static java.time.temporal.ChronoField.DAY_OF_MONTH; +import static java.time.temporal.ChronoField.HOUR_OF_DAY; +import static java.time.temporal.ChronoField.MINUTE_OF_DAY; +import static java.time.temporal.ChronoField.MONTH_OF_YEAR; +import static java.time.temporal.ChronoField.NANO_OF_SECOND; +import static java.time.temporal.ChronoField.SECOND_OF_DAY; + enum DateFormat { Iso8601 { @Override @@ -70,22 +81,37 @@ private long parseMillis(String date) { } }, Java { + private final List FIELDS = + Arrays.asList(NANO_OF_SECOND, SECOND_OF_DAY, MINUTE_OF_DAY, HOUR_OF_DAY, DAY_OF_MONTH, MONTH_OF_YEAR); + @Override Function getFunction(String format, DateTimeZone timezone, Locale locale) { - // support the 6.x BWC compatible way of parsing java 8 dates if (format.startsWith("8")) { format = format.substring(1); } + ZoneId zoneId = DateUtils.dateTimeZoneToZoneId(timezone); int year = LocalDate.now(ZoneOffset.UTC).getYear(); DateFormatter formatter = DateFormatter.forPattern(format) .withLocale(locale) - .withZone(DateUtils.dateTimeZoneToZoneId(timezone)); + .withZone(zoneId); return text -> { - ZonedDateTime defaultZonedDateTime = Instant.EPOCH.atZone(ZoneOffset.UTC).withYear(year); TemporalAccessor accessor = formatter.parse(text); - long millis = DateFormatters.toZonedDateTime(accessor, defaultZonedDateTime).toInstant().toEpochMilli(); + // if there is no year, we fall back to the current one and + // fill the rest of the date up with the parsed date + if (accessor.isSupported(ChronoField.YEAR) == false) { + ZonedDateTime newTime = Instant.EPOCH.atZone(ZoneOffset.UTC).withYear(year); + for (ChronoField field : FIELDS) { + if (accessor.isSupported(field)) { + newTime = newTime.with(field, accessor.get(field)); + } + } + + accessor = newTime.withZoneSameLocal(zoneId); + } + + long millis = DateFormatters.from(accessor).toInstant().toEpochMilli(); return new DateTime(millis, timezone); }; } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java index 27904a5586e7e..32874aa6a5776 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java @@ -19,29 +19,43 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.test.ESTestCase; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import java.time.Instant; import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.Locale; import java.util.function.Function; +import static org.hamcrest.Matchers.is; import static org.hamcrest.core.IsEqual.equalTo; public class DateFormatTests extends ESTestCase { - public void testParseJoda() { - Function jodaFunction = DateFormat.Java.getFunction("MMM dd HH:mm:ss Z", + public void testParseJava() { + Function javaFunction = DateFormat.Java.getFunction("MMM dd HH:mm:ss Z", DateTimeZone.forOffsetHours(-8), Locale.ENGLISH); - assertThat(Instant.ofEpochMilli(jodaFunction.apply("Nov 24 01:29:01 -0800").getMillis()) + assertThat(Instant.ofEpochMilli(javaFunction.apply("Nov 24 01:29:01 -0800").getMillis()) .atZone(ZoneId.of("GMT-8")) .format(DateTimeFormatter.ofPattern("MM dd HH:mm:ss", Locale.ENGLISH)), equalTo("11 24 01:29:01")); } + public void testParseJavaDefaultYear() { + String format = randomFrom("8dd/MM", "dd/MM"); + DateTimeZone timezone = DateUtils.zoneIdToDateTimeZone(ZoneId.of("Europe/Amsterdam")); + Function javaFunction = DateFormat.Java.getFunction(format, timezone, Locale.ENGLISH); + int year = ZonedDateTime.now(ZoneOffset.UTC).getYear(); + DateTime dateTime = javaFunction.apply("12/06"); + assertThat(dateTime.getYear(), is(year)); + assertThat(dateTime.toString(), is(year + "-06-12T00:00:00.000+02:00")); + } + public void testParseUnixMs() { assertThat(DateFormat.UnixMs.getFunction(null, DateTimeZone.UTC, null).apply("1000500").getMillis(), equalTo(1000500L)); } diff --git a/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java b/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java index 0b92955583f7f..6f16e4bc71a71 100644 --- a/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java +++ b/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java @@ -20,11 +20,13 @@ package org.elasticsearch.common.time; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.SuppressForbidden; -import java.time.DateTimeException; import java.time.DayOfWeek; import java.time.Instant; import java.time.LocalDate; +import java.time.LocalTime; +import java.time.Year; import java.time.ZoneId; import java.time.ZoneOffset; import java.time.ZonedDateTime; @@ -1550,105 +1552,91 @@ static JavaDateFormatter merge(String pattern, List formatters) { dateTimeFormatters.toArray(new DateTimeFormatter[0])); } - private static final ZonedDateTime EPOCH_ZONED_DATE_TIME = Instant.EPOCH.atZone(ZoneOffset.UTC); + private static final LocalDate LOCALDATE_EPOCH = LocalDate.of(1970, 1, 1); - public static ZonedDateTime toZonedDateTime(TemporalAccessor accessor) { - return toZonedDateTime(accessor, EPOCH_ZONED_DATE_TIME); - } - - public static ZonedDateTime toZonedDateTime(TemporalAccessor accessor, ZonedDateTime defaults) { - try { - return ZonedDateTime.from(accessor); - } catch (DateTimeException e ) { + /** + * Convert a temporal accessor to a zoned date time object - as performant as possible. + * The .from() methods from the JDK are throwing exceptions when for example ZonedDateTime.from(accessor) + * or Instant.from(accessor). This results in a huge performance penalty and should be prevented + * This method prevents exceptions by querying the accessor for certain capabilities + * and then act on it accordingly + * + * This action assumes that we can reliably fall back to some defaults if not all parts of a + * zoned date time are set + * + * - If a zoned date time is passed, it is returned + * - If no timezone is found, ZoneOffset.UTC is used + * - If we find a time and a date, converting to a ZonedDateTime is straight forward, + * no defaults will be applied + * - If an accessor only containing of seconds and nanos is found (like epoch_millis/second) + * an Instant is created out of that, that becomes a ZonedDateTime with a time zone + * - If no time is given, the start of the day is used + * - If no month of the year is found, the first day of the year is used + * - If an iso based weekyear is found, but not week is specified, the first monday + * of the new year is chosen (reataining BWC to joda time) + * - If an iso based weekyear is found and an iso based weekyear week, the start + * of the day is used + * + * @param accessor The accessor returned from a parser + * + * @return The converted zoned date time + */ + public static ZonedDateTime from(TemporalAccessor accessor) { + if (accessor instanceof ZonedDateTime) { + return (ZonedDateTime) accessor; } - ZonedDateTime result = defaults; - - // special case epoch seconds - if (accessor.isSupported(ChronoField.INSTANT_SECONDS)) { - result = result.with(ChronoField.INSTANT_SECONDS, accessor.getLong(ChronoField.INSTANT_SECONDS)); - if (accessor.isSupported(ChronoField.NANO_OF_SECOND)) { - result = result.with(ChronoField.NANO_OF_SECOND, accessor.getLong(ChronoField.NANO_OF_SECOND)); - } - return result; + ZoneId zoneId = accessor.query(TemporalQueries.zone()); + if (zoneId == null) { + zoneId = ZoneOffset.UTC; } - - // try to set current year - if (accessor.isSupported(ChronoField.YEAR)) { - result = result.with(ChronoField.YEAR, accessor.getLong(ChronoField.YEAR)); - } else if (accessor.isSupported(ChronoField.YEAR_OF_ERA)) { - result = result.with(ChronoField.YEAR_OF_ERA, accessor.getLong(ChronoField.YEAR_OF_ERA)); + + LocalDate localDate = accessor.query(TemporalQueries.localDate()); + LocalTime localTime = accessor.query(TemporalQueries.localTime()); + boolean isLocalDateSet = localDate != null; + boolean isLocalTimeSet = localTime != null; + + // the first two cases are the most common, so this allows us to exit early when parsing dates + if (isLocalDateSet && isLocalTimeSet) { + return of(localDate, localTime, zoneId); + } else if (accessor.isSupported(ChronoField.INSTANT_SECONDS) && accessor.isSupported(NANO_OF_SECOND)) { + return Instant.from(accessor).atZone(zoneId); + } else if (isLocalDateSet) { + return localDate.atStartOfDay(zoneId); + } else if (isLocalTimeSet) { + return of(LOCALDATE_EPOCH, localTime, zoneId); + } else if (accessor.isSupported(ChronoField.YEAR)) { + if (accessor.isSupported(MONTH_OF_YEAR)) { + return getFirstOfMonth(accessor).atStartOfDay(zoneId); + } else { + return Year.of(accessor.get(ChronoField.YEAR)).atDay(1).atStartOfDay(zoneId); + } } else if (accessor.isSupported(WeekFields.ISO.weekBasedYear())) { if (accessor.isSupported(WeekFields.ISO.weekOfWeekBasedYear())) { - return LocalDate.from(result) - .with(WeekFields.ISO.weekBasedYear(), accessor.getLong(WeekFields.ISO.weekBasedYear())) - .withDayOfMonth(1) // makes this compatible with joda + return Year.of(accessor.get(WeekFields.ISO.weekBasedYear())) + .atDay(1) .with(WeekFields.ISO.weekOfWeekBasedYear(), accessor.getLong(WeekFields.ISO.weekOfWeekBasedYear())) - .atStartOfDay(ZoneOffset.UTC); + .atStartOfDay(zoneId); } else { - return LocalDate.from(result) - .with(WeekFields.ISO.weekBasedYear(), accessor.getLong(WeekFields.ISO.weekBasedYear())) - // this exists solely to be BWC compatible with joda -// .with(TemporalAdjusters.nextOrSame(DayOfWeek.MONDAY)) + return Year.of(accessor.get(WeekFields.ISO.weekBasedYear())) + .atDay(1) .with(TemporalAdjusters.firstInMonth(DayOfWeek.MONDAY)) - .atStartOfDay(defaults.getZone()); -// return result.withHour(0).withMinute(0).withSecond(0) -// .with(WeekFields.ISO.weekBasedYear(), 0) -// .with(WeekFields.ISO.weekBasedYear(), accessor.getLong(WeekFields.ISO.weekBasedYear())); -// return ((ZonedDateTime) tmp).with(WeekFields.ISO.weekOfWeekBasedYear(), 1); - } - } else if (accessor.isSupported(IsoFields.WEEK_BASED_YEAR)) { - // special case weekbased year - result = result.with(IsoFields.WEEK_BASED_YEAR, accessor.getLong(IsoFields.WEEK_BASED_YEAR)); - if (accessor.isSupported(IsoFields.WEEK_OF_WEEK_BASED_YEAR)) { - result = result.with(IsoFields.WEEK_OF_WEEK_BASED_YEAR, accessor.getLong(IsoFields.WEEK_OF_WEEK_BASED_YEAR)); + .atStartOfDay(zoneId); } - return result; - } - - // month - if (accessor.isSupported(ChronoField.MONTH_OF_YEAR)) { - result = result.with(ChronoField.MONTH_OF_YEAR, accessor.getLong(ChronoField.MONTH_OF_YEAR)); } - // day of month - if (accessor.isSupported(ChronoField.DAY_OF_MONTH)) { - result = result.with(ChronoField.DAY_OF_MONTH, accessor.getLong(ChronoField.DAY_OF_MONTH)); - } - - // hour - if (accessor.isSupported(ChronoField.HOUR_OF_DAY)) { - result = result.with(ChronoField.HOUR_OF_DAY, accessor.getLong(ChronoField.HOUR_OF_DAY)); - } - - // minute - if (accessor.isSupported(ChronoField.MINUTE_OF_HOUR)) { - result = result.with(ChronoField.MINUTE_OF_HOUR, accessor.getLong(ChronoField.MINUTE_OF_HOUR)); - } - - // second - if (accessor.isSupported(ChronoField.SECOND_OF_MINUTE)) { - result = result.with(ChronoField.SECOND_OF_MINUTE, accessor.getLong(ChronoField.SECOND_OF_MINUTE)); - } - - if (accessor.isSupported(ChronoField.OFFSET_SECONDS)) { - result = result.withZoneSameLocal(ZoneOffset.ofTotalSeconds(accessor.get(ChronoField.OFFSET_SECONDS))); - } - - // millis - if (accessor.isSupported(ChronoField.MILLI_OF_SECOND)) { - result = result.with(ChronoField.MILLI_OF_SECOND, accessor.getLong(ChronoField.MILLI_OF_SECOND)); - } - - if (accessor.isSupported(ChronoField.NANO_OF_SECOND)) { - result = result.with(ChronoField.NANO_OF_SECOND, accessor.getLong(ChronoField.NANO_OF_SECOND)); - } + // we should not reach this piece of code, everything being parsed we should be able to + // convert to a zoned date time! If not, we have to extend the above methods + throw new IllegalArgumentException("temporal accessor [" + accessor + "] cannot be converted to zoned date time"); + } - ZoneId zoneOffset = accessor.query(TemporalQueries.zone()); - if (zoneOffset != null) { - result = result.withZoneSameLocal(zoneOffset); - } + @SuppressForbidden(reason = "ZonedDateTime.of is fine here") + private static ZonedDateTime of(LocalDate localDate, LocalTime localTime, ZoneId zoneId) { + return ZonedDateTime.of(localDate, localTime, zoneId); + } - return result; + @SuppressForbidden(reason = "LocalDate.of is fine here") + private static LocalDate getFirstOfMonth(TemporalAccessor accessor) { + return LocalDate.of(accessor.get(ChronoField.YEAR), accessor.get(MONTH_OF_YEAR), 1); } } diff --git a/server/src/main/java/org/elasticsearch/common/time/JavaDateMathParser.java b/server/src/main/java/org/elasticsearch/common/time/JavaDateMathParser.java index 9ee390ba391a7..05e1e75efca39 100644 --- a/server/src/main/java/org/elasticsearch/common/time/JavaDateMathParser.java +++ b/server/src/main/java/org/elasticsearch/common/time/JavaDateMathParser.java @@ -218,7 +218,7 @@ private Instant parseDateTime(String value, ZoneId timeZone, boolean roundUpIfNo DateTimeFormatter formatter = roundUpIfNoTime ? this.roundUpFormatter : this.formatter; try { if (timeZone == null) { - return DateFormatters.toZonedDateTime(formatter.parse(value)).toInstant(); + return DateFormatters.from(formatter.parse(value)).toInstant(); } else { TemporalAccessor accessor = formatter.parse(value); ZoneId zoneId = TemporalQueries.zone().queryFrom(accessor); @@ -226,7 +226,7 @@ private Instant parseDateTime(String value, ZoneId timeZone, boolean roundUpIfNo timeZone = zoneId; } - return DateFormatters.toZonedDateTime(accessor).withZoneSameLocal(timeZone).toInstant(); + return DateFormatters.from(accessor).withZoneSameLocal(timeZone).toInstant(); } } catch (DateTimeParseException e) { throw new ElasticsearchParseException("failed to parse date field [{}] with format [{}]: [{}]", diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 0dcf52d5e54f2..0af9443a64131 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -247,7 +247,7 @@ protected DateMathParser dateMathParser() { } long parse(String value) { - return DateFormatters.toZonedDateTime(dateTimeFormatter().parse(value)).toInstant().toEpochMilli(); + return DateFormatters.from(dateTimeFormatter().parse(value)).toInstant().toEpochMilli(); } @Override diff --git a/server/src/test/java/org/elasticsearch/common/RoundingTests.java b/server/src/test/java/org/elasticsearch/common/RoundingTests.java index 9bc7c10abd8c8..a809131b932e2 100644 --- a/server/src/test/java/org/elasticsearch/common/RoundingTests.java +++ b/server/src/test/java/org/elasticsearch/common/RoundingTests.java @@ -730,7 +730,7 @@ private static long time(String time) { private static long time(String time, ZoneId zone) { TemporalAccessor accessor = DateFormatter.forPattern("date_optional_time").withZone(zone).parse(time); - return DateFormatters.toZonedDateTime(accessor).toInstant().toEpochMilli(); + return DateFormatters.from(accessor).toInstant().toEpochMilli(); } private static Matcher isDate(final long expected, ZoneId tz) { diff --git a/server/src/test/java/org/elasticsearch/common/joda/JavaJodaTimeDuellingTests.java b/server/src/test/java/org/elasticsearch/common/joda/JavaJodaTimeDuellingTests.java index 423592d6d18d7..a0fcf988ca811 100644 --- a/server/src/test/java/org/elasticsearch/common/joda/JavaJodaTimeDuellingTests.java +++ b/server/src/test/java/org/elasticsearch/common/joda/JavaJodaTimeDuellingTests.java @@ -688,7 +688,7 @@ private void assertSameDate(String input, String format, DateFormatter jodaForma DateTime jodaDateTime = jodaFormatter.parseJoda(input); TemporalAccessor javaTimeAccessor = javaFormatter.parse(input); - ZonedDateTime zonedDateTime = DateFormatters.toZonedDateTime(javaTimeAccessor); + ZonedDateTime zonedDateTime = DateFormatters.from(javaTimeAccessor); String msg = String.format(Locale.ROOT, "Input [%s] Format [%s] Joda [%s], Java [%s]", input, format, jodaDateTime, DateTimeFormatter.ISO_INSTANT.format(zonedDateTime.toInstant())); diff --git a/server/src/test/java/org/elasticsearch/common/time/DateFormattersTests.java b/server/src/test/java/org/elasticsearch/common/time/DateFormattersTests.java index 96ef39e430178..90a9a76e6a4f9 100644 --- a/server/src/test/java/org/elasticsearch/common/time/DateFormattersTests.java +++ b/server/src/test/java/org/elasticsearch/common/time/DateFormattersTests.java @@ -84,14 +84,14 @@ public void testEpochSecondParser() { public void testEpochMilliParsersWithDifferentFormatters() { DateFormatter formatter = DateFormatter.forPattern("strict_date_optional_time||epoch_millis"); TemporalAccessor accessor = formatter.parse("123"); - assertThat(DateFormatters.toZonedDateTime(accessor).toInstant().toEpochMilli(), is(123L)); + assertThat(DateFormatters.from(accessor).toInstant().toEpochMilli(), is(123L)); assertThat(formatter.pattern(), is("strict_date_optional_time||epoch_millis")); } public void testParsersWithMultipleInternalFormats() throws Exception { - ZonedDateTime first = DateFormatters.toZonedDateTime( + ZonedDateTime first = DateFormatters.from( DateFormatters.forPattern("strict_date_optional_time_nanos").parse("2018-05-15T17:14:56+0100")); - ZonedDateTime second = DateFormatters.toZonedDateTime( + ZonedDateTime second = DateFormatters.from( DateFormatters.forPattern("strict_date_optional_time_nanos").parse("2018-05-15T17:14:56+01:00")); assertThat(first, is(second)); } @@ -163,7 +163,7 @@ public void testRoundupFormatterWithEpochDates() { assertRoundupFormatter("epoch_millis", "1234567890", 1234567890L); // also check nanos of the epoch_millis formatter if it is rounded up to the nano second DateTimeFormatter roundUpFormatter = ((JavaDateFormatter) DateFormatter.forPattern("8epoch_millis")).getRoundupParser(); - Instant epochMilliInstant = DateFormatters.toZonedDateTime(roundUpFormatter.parse("1234567890")).toInstant(); + Instant epochMilliInstant = DateFormatters.from(roundUpFormatter.parse("1234567890")).toInstant(); assertThat(epochMilliInstant.getLong(ChronoField.NANO_OF_SECOND), is(890_999_999L)); assertRoundupFormatter("strict_date_optional_time||epoch_millis", "2018-10-10T12:13:14.123Z", 1539173594123L); @@ -175,7 +175,7 @@ public void testRoundupFormatterWithEpochDates() { assertRoundupFormatter("epoch_second", "1234567890", 1234567890999L); // also check nanos of the epoch_millis formatter if it is rounded up to the nano second DateTimeFormatter epochSecondRoundupParser = ((JavaDateFormatter) DateFormatter.forPattern("8epoch_second")).getRoundupParser(); - Instant epochSecondInstant = DateFormatters.toZonedDateTime(epochSecondRoundupParser.parse("1234567890")).toInstant(); + Instant epochSecondInstant = DateFormatters.from(epochSecondRoundupParser.parse("1234567890")).toInstant(); assertThat(epochSecondInstant.getLong(ChronoField.NANO_OF_SECOND), is(999_999_999L)); assertRoundupFormatter("strict_date_optional_time||epoch_second", "2018-10-10T12:13:14.123Z", 1539173594123L); @@ -189,7 +189,7 @@ private void assertRoundupFormatter(String format, String input, long expectedMi JavaDateFormatter dateFormatter = (JavaDateFormatter) DateFormatter.forPattern(format); dateFormatter.parse(input); DateTimeFormatter roundUpFormatter = dateFormatter.getRoundupParser(); - long millis = DateFormatters.toZonedDateTime(roundUpFormatter.parse(input)).toInstant().toEpochMilli(); + long millis = DateFormatters.from(roundUpFormatter.parse(input)).toInstant().toEpochMilli(); assertThat(millis, is(expectedMilliSeconds)); } diff --git a/server/src/test/java/org/elasticsearch/common/time/JavaDateMathParserTests.java b/server/src/test/java/org/elasticsearch/common/time/JavaDateMathParserTests.java index 2b8d89bc68bae..cbd102451918d 100644 --- a/server/src/test/java/org/elasticsearch/common/time/JavaDateMathParserTests.java +++ b/server/src/test/java/org/elasticsearch/common/time/JavaDateMathParserTests.java @@ -138,12 +138,12 @@ public void testRoundingPreservesEpochAsBaseDate() { // If a user only specifies times, then the date needs to always be 1970-01-01 regardless of rounding DateFormatter formatter = DateFormatters.forPattern("HH:mm:ss"); DateMathParser parser = formatter.toDateMathParser(); - ZonedDateTime zonedDateTime = DateFormatters.toZonedDateTime(formatter.parse("04:52:20")); + ZonedDateTime zonedDateTime = DateFormatters.from(formatter.parse("04:52:20")); assertThat(zonedDateTime.getYear(), is(1970)); Instant millisStart = zonedDateTime.toInstant(); assertEquals(millisStart, parser.parse("04:52:20", () -> 0, false, (ZoneId) null)); // due to rounding up, we have to add the number of milliseconds here manually - long millisEnd = DateFormatters.toZonedDateTime(formatter.parse("04:52:20")).toInstant().toEpochMilli() + 999; + long millisEnd = DateFormatters.from(formatter.parse("04:52:20")).toInstant().toEpochMilli() + 999; assertEquals(millisEnd, parser.parse("04:52:20", () -> 0, true, (ZoneId) null).toEpochMilli()); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java index d4058d50f74a2..92178e93d212b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java @@ -113,9 +113,9 @@ public void testIsFieldWithinQuery() throws IOException { Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null)); long instant1 = - DateFormatters.toZonedDateTime(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse("2015-10-12")).toInstant().toEpochMilli(); + DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse("2015-10-12")).toInstant().toEpochMilli(); long instant2 = - DateFormatters.toZonedDateTime(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse("2016-04-03")).toInstant().toEpochMilli(); + DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse("2016-04-03")).toInstant().toEpochMilli(); Document doc = new Document(); LongPoint field = new LongPoint("my_date", instant1); doc.add(field); @@ -142,7 +142,7 @@ public void testIsFieldWithinQuery() throws IOException { public void testValueFormat() { MappedFieldType ft = createDefaultFieldType(); - long instant = DateFormatters.toZonedDateTime(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse("2015-10-12T14:10:55")) + long instant = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse("2015-10-12T14:10:55")) .toInstant().toEpochMilli(); assertEquals("2015-10-12T14:10:55.000Z", @@ -155,7 +155,7 @@ public void testValueFormat() { ft.docValueFormat(null, ZoneOffset.UTC).parseLong("2015-10-12T14:10:55", false, null)); assertEquals(instant + 999, ft.docValueFormat(null, ZoneOffset.UTC).parseLong("2015-10-12T14:10:55", true, null)); - long i = DateFormatters.toZonedDateTime(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse("2015-10-13")).toInstant().toEpochMilli(); + long i = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse("2015-10-13")).toInstant().toEpochMilli(); assertEquals(i - 1, ft.docValueFormat(null, ZoneOffset.UTC).parseLong("2015-10-12||/d", true, null)); } @@ -176,7 +176,7 @@ public void testTermQuery() { MappedFieldType ft = createDefaultFieldType(); ft.setName("field"); String date = "2015-10-12T14:10:55"; - long instant = DateFormatters.toZonedDateTime(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)).toInstant().toEpochMilli(); + long instant = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)).toInstant().toEpochMilli(); ft.setIndexOptions(IndexOptions.DOCS); Query expected = new IndexOrDocValuesQuery( LongPoint.newRangeQuery("field", instant, instant + 999), @@ -199,9 +199,9 @@ public void testRangeQuery() throws IOException { ft.setName("field"); String date1 = "2015-10-12T14:10:55"; String date2 = "2016-04-28T11:33:52"; - long instant1 = DateFormatters.toZonedDateTime(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date1)).toInstant().toEpochMilli(); + long instant1 = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date1)).toInstant().toEpochMilli(); long instant2 = - DateFormatters.toZonedDateTime(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date2)).toInstant().toEpochMilli() + 999; + DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date2)).toInstant().toEpochMilli() + 999; ft.setIndexOptions(IndexOptions.DOCS); Query expected = new IndexOrDocValuesQuery( LongPoint.newRangeQuery("field", instant1, instant2), diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java index c59be546acd1a..a79797f6c822e 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java @@ -91,7 +91,7 @@ private ZonedDateTime date(int month, int day) { } private ZonedDateTime date(String date) { - return DateFormatters.toZonedDateTime(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)); + return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)); } private static String format(ZonedDateTime date, String pattern) { @@ -1191,10 +1191,10 @@ public void testSingleValueFieldWithExtendedBoundsOffset() throws Exception { List builders = new ArrayList<>(); DateFormatter formatter = DateFormatter.forPattern("date_optional_time"); - builders.add(indexDoc(index, DateFormatters.toZonedDateTime(formatter.parse("2016-01-03T08:00:00.000Z")), 1)); - builders.add(indexDoc(index, DateFormatters.toZonedDateTime(formatter.parse("2016-01-03T08:00:00.000Z")), 2)); - builders.add(indexDoc(index, DateFormatters.toZonedDateTime(formatter.parse("2016-01-06T08:00:00.000Z")), 3)); - builders.add(indexDoc(index, DateFormatters.toZonedDateTime(formatter.parse("2016-01-06T08:00:00.000Z")), 4)); + builders.add(indexDoc(index, DateFormatters.from(formatter.parse("2016-01-03T08:00:00.000Z")), 1)); + builders.add(indexDoc(index, DateFormatters.from(formatter.parse("2016-01-03T08:00:00.000Z")), 2)); + builders.add(indexDoc(index, DateFormatters.from(formatter.parse("2016-01-06T08:00:00.000Z")), 3)); + builders.add(indexDoc(index, DateFormatters.from(formatter.parse("2016-01-06T08:00:00.000Z")), 4)); indexRandom(true, builders); ensureSearchable(index); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramOffsetIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramOffsetIT.java index 080c4faffd696..0fab4073e334b 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramOffsetIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramOffsetIT.java @@ -54,7 +54,7 @@ public class DateHistogramOffsetIT extends ESIntegTestCase { private static final DateFormatter FORMATTER = DateFormatter.forPattern(DATE_FORMAT); private ZonedDateTime date(String date) { - return DateFormatters.toZonedDateTime(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)); + return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)); } @Before diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java index 5f219ee6be948..52cff012b6473 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java @@ -1835,6 +1835,6 @@ private static Map> createDocument(Object... fields) { } private static long asLong(String dateTime) { - return DateFormatters.toZonedDateTime(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); + return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java index 2fbf60a3ddccb..3ce74b04e23b8 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java @@ -475,6 +475,6 @@ private void executeTestCase(boolean reduced, Query query, List dataset, } private static long asLong(String dateTime) { - return DateFormatters.toZonedDateTime(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); + return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramTests.java index c65b21ef72d32..1a639552ae4be 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramTests.java @@ -143,14 +143,14 @@ public void testRewriteTimeZone() throws IOException { try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) { - long millis1 = DateFormatters.toZonedDateTime(format.parse("2018-03-11T11:55:00")).toInstant().toEpochMilli(); + long millis1 = DateFormatters.from(format.parse("2018-03-11T11:55:00")).toInstant().toEpochMilli(); w.addDocument(documentForDate(DATE_FIELD_NAME, millis1)); - long millis2 = DateFormatters.toZonedDateTime(format.parse("2017-10-30T18:13:00")).toInstant().toEpochMilli(); + long millis2 = DateFormatters.from(format.parse("2017-10-30T18:13:00")).toInstant().toEpochMilli(); w.addDocument(documentForDate(DATE_FIELD_NAME, millis2)); try (IndexReader readerThatDoesntCross = DirectoryReader.open(w)) { - long millis3 = DateFormatters.toZonedDateTime(format.parse("2018-03-25T02:44:00")).toInstant().toEpochMilli(); + long millis3 = DateFormatters.from(format.parse("2018-03-25T02:44:00")).toInstant().toEpochMilli(); w.addDocument(documentForDate(DATE_FIELD_NAME, millis3)); try (IndexReader readerThatCrosses = DirectoryReader.open(w)) { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketAggregatorTests.java index 3ed1a15603e84..627ca9c0af9bb 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketAggregatorTests.java @@ -140,6 +140,6 @@ public void testSameAggNames() throws IOException { } private static long asLong(String dateTime) { - return DateFormatters.toZonedDateTime(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); + return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumAggregatorTests.java index 4b23304e642c0..e3475be5773e8 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumAggregatorTests.java @@ -366,6 +366,6 @@ private void executeTestCase(Query query, AggregationBuilder aggBuilder, Consume } private static long asLong(String dateTime) { - return DateFormatters.toZonedDateTime(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); + return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DateDerivativeIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DateDerivativeIT.java index db1ee6ab18916..395d7498732c3 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DateDerivativeIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DateDerivativeIT.java @@ -200,11 +200,11 @@ public void testSingleValuedFieldNormalised_timeZone_CET_DstStart() throws Excep ZoneId timezone = ZoneId.of("CET"); DateFormatter formatter = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ss").withZone(timezone); // epoch millis: 1332547200000 - addNTimes(1, IDX_DST_START, DateFormatters.toZonedDateTime(formatter.parse("2012-03-24T01:00:00")), builders); + addNTimes(1, IDX_DST_START, DateFormatters.from(formatter.parse("2012-03-24T01:00:00")), builders); // day with dst shift, only 23h long - addNTimes(2, IDX_DST_START, DateFormatters.toZonedDateTime(formatter.parse("2012-03-25T01:00:00")), builders); - addNTimes(3, IDX_DST_START, DateFormatters.toZonedDateTime(formatter.parse("2012-03-26T01:00:00")), builders); - addNTimes(4, IDX_DST_START, DateFormatters.toZonedDateTime(formatter.parse("2012-03-27T01:00:00")), builders); + addNTimes(2, IDX_DST_START, DateFormatters.from(formatter.parse("2012-03-25T01:00:00")), builders); + addNTimes(3, IDX_DST_START, DateFormatters.from(formatter.parse("2012-03-26T01:00:00")), builders); + addNTimes(4, IDX_DST_START, DateFormatters.from(formatter.parse("2012-03-27T01:00:00")), builders); indexRandom(true, builders); ensureSearchable(); @@ -251,11 +251,11 @@ public void testSingleValuedFieldNormalised_timeZone_CET_DstEnd() throws Excepti List builders = new ArrayList<>(); DateFormatter formatter = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ss").withZone(timezone); - addNTimes(1, IDX_DST_END, DateFormatters.toZonedDateTime(formatter.parse("2012-10-27T01:00:00")), builders); + addNTimes(1, IDX_DST_END, DateFormatters.from(formatter.parse("2012-10-27T01:00:00")), builders); // day with dst shift -1h, 25h long - addNTimes(2, IDX_DST_END, DateFormatters.toZonedDateTime(formatter.parse("2012-10-28T01:00:00")), builders); - addNTimes(3, IDX_DST_END, DateFormatters.toZonedDateTime(formatter.parse("2012-10-29T01:00:00")), builders); - addNTimes(4, IDX_DST_END, DateFormatters.toZonedDateTime(formatter.parse("2012-10-30T01:00:00")), builders); + addNTimes(2, IDX_DST_END, DateFormatters.from(formatter.parse("2012-10-28T01:00:00")), builders); + addNTimes(3, IDX_DST_END, DateFormatters.from(formatter.parse("2012-10-29T01:00:00")), builders); + addNTimes(4, IDX_DST_END, DateFormatters.from(formatter.parse("2012-10-30T01:00:00")), builders); indexRandom(true, builders); ensureSearchable(); @@ -304,11 +304,11 @@ public void testSingleValuedFieldNormalised_timeZone_AsiaKathmandu() throws Exce List builders = new ArrayList<>(); DateFormatter formatter = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ss").withZone(timezone); - addNTimes(1, IDX_DST_KATHMANDU, DateFormatters.toZonedDateTime(formatter.parse("1985-12-31T22:30:00")), builders); + addNTimes(1, IDX_DST_KATHMANDU, DateFormatters.from(formatter.parse("1985-12-31T22:30:00")), builders); // the shift happens during the next bucket, which includes the 45min that do not start on the full hour - addNTimes(2, IDX_DST_KATHMANDU, DateFormatters.toZonedDateTime(formatter.parse("1985-12-31T23:30:00")), builders); - addNTimes(3, IDX_DST_KATHMANDU, DateFormatters.toZonedDateTime(formatter.parse("1986-01-01T01:30:00")), builders); - addNTimes(4, IDX_DST_KATHMANDU, DateFormatters.toZonedDateTime(formatter.parse("1986-01-01T02:30:00")), builders); + addNTimes(2, IDX_DST_KATHMANDU, DateFormatters.from(formatter.parse("1985-12-31T23:30:00")), builders); + addNTimes(3, IDX_DST_KATHMANDU, DateFormatters.from(formatter.parse("1986-01-01T01:30:00")), builders); + addNTimes(4, IDX_DST_KATHMANDU, DateFormatters.from(formatter.parse("1986-01-01T02:30:00")), builders); indexRandom(true, builders); ensureSearchable(); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnUnitTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnUnitTests.java index b4ae26d5f13df..1368db5ab71e6 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnUnitTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnUnitTests.java @@ -161,7 +161,7 @@ public double execute(Map params, double[] values) { } private static long asLong(String dateTime) { - return DateFormatters.toZonedDateTime(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); + return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); } /** diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/DateUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/DateUtils.java index 9cec2b4cc7c1f..2aa41041728a2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/DateUtils.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/DateUtils.java @@ -23,7 +23,7 @@ public static long endOfTheDay(String date) { // Try parsing using complete date/time format return dateTimeFormatter.parseMillis(date); } catch (ElasticsearchParseException | IllegalArgumentException ex) { - ZonedDateTime dateTime = DateFormatters.toZonedDateTime(dateOnlyFormatter.parse(date)); + ZonedDateTime dateTime = DateFormatters.from(dateOnlyFormatter.parse(date)); dateTime.with(ChronoField.MILLI_OF_DAY, ChronoField.MILLI_OF_DAY.range().getMaximum()); return dateTime.toInstant().toEpochMilli(); } @@ -35,7 +35,7 @@ public static long beginningOfTheDay(String date) { return dateTimeFormatter.parseMillis(date); } catch (ElasticsearchParseException | IllegalArgumentException ex) { // Fall back to the date only format - return DateFormatters.toZonedDateTime(dateOnlyFormatter.parse(date)).toInstant().toEpochMilli(); + return DateFormatters.from(dateOnlyFormatter.parse(date)).toInstant().toEpochMilli(); } } } From e0d5de33da5703bbcffe4dd179947757499e2bc0 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Wed, 30 Jan 2019 23:57:56 -0800 Subject: [PATCH 086/100] fix DateIndexNameProcessorTests offset pattern (#38069) `XX` was being used to represent an offset pattern, it should be `ZZ` Fixes #38067. --- .../ingest/common/DateIndexNameProcessorTests.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java index 3ac885d8680e9..63d3e0416cd2c 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java @@ -79,11 +79,10 @@ public void testUnix()throws Exception { assertThat(document.getSourceAndMetadata().get("_index"), equalTo("")); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/38067") public void testTemplatedFields() throws Exception { String indexNamePrefix = randomAlphaOfLength(10); String dateRounding = randomFrom("y", "M", "w", "d", "h", "m", "s"); - String indexNameFormat = randomFrom("yyyy-MM-dd'T'HH:mm:ss.SSSXX", "yyyyMMdd", "MM/dd/yyyy"); + String indexNameFormat = randomFrom("yyyy-MM-dd'T'HH:mm:ss.SSSZZ", "yyyyMMdd", "MM/dd/yyyy"); String date = Integer.toString(randomInt()); Function dateTimeFunction = DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null); From eadcb5f0f8f8e27a5719037968bf43ce5c03ccb8 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 31 Jan 2019 08:00:11 +0000 Subject: [PATCH 087/100] Fix size of rolling-upgrade bootstrap config (#38031) Zen2 nodes will bootstrap themselves once they believe there to be no remaining Zen1 master-eligible nodes in the cluster, as long as minimum_master_nodes is satisfied. Today the bootstrap configuration comprises just the ids of the known master-eligible nodes, and this might be too small to be safe. For instance, if there are 5 master-eligible nodes (so that minimum_master_nodes is 3) then the bootstrap configuration could comprise just 3 nodes, of which 2 form a quorum, and this does not intersect other quorums that might arise, leading to a split-brain. This commit fixes this by expanding the bootstrap configuration so that its quorums satisfy minimum_master_nodes, by adding some of the IDs of the other master-eligible nodes in the last-published cluster state. --- .../coordination/DiscoveryUpgradeService.java | 28 ++++++-- .../cluster/coordination/Zen1IT.java | 66 +++++++++++++++++++ 2 files changed, 90 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeService.java index 56102704848c8..e793e407e1a89 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeService.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeService.java @@ -47,6 +47,8 @@ import org.elasticsearch.transport.TransportService; import java.io.IOException; +import java.util.HashSet; +import java.util.Iterator; import java.util.Optional; import java.util.Set; import java.util.function.BooleanSupplier; @@ -130,7 +132,11 @@ public void activate(Optional lastKnownLeader, ClusterState lastA : lastAcceptedClusterState.getMinimumMasterNodesOnPublishingMaster(); assert joiningRound == null : joiningRound; - joiningRound = new JoiningRound(enableUnsafeBootstrappingOnUpgrade && lastKnownLeader.isPresent(), minimumMasterNodes); + final Set knownMasterNodeIds = new HashSet<>(); + lastAcceptedClusterState.nodes().getMasterNodes().forEach(c -> knownMasterNodeIds.add(c.key)); + + joiningRound + = new JoiningRound(enableUnsafeBootstrappingOnUpgrade && lastKnownLeader.isPresent(), minimumMasterNodes, knownMasterNodeIds); joiningRound.scheduleNextAttempt(); } @@ -168,10 +174,12 @@ void countDown() { private class JoiningRound { private final boolean upgrading; private final int minimumMasterNodes; + private final Set knownMasterNodeIds; - JoiningRound(boolean upgrading, int minimumMasterNodes) { + JoiningRound(boolean upgrading, int minimumMasterNodes, Set knownMasterNodeIds) { this.upgrading = upgrading; this.minimumMasterNodes = minimumMasterNodes; + this.knownMasterNodeIds = knownMasterNodeIds; } private boolean isRunning() { @@ -210,8 +218,20 @@ public void run() { // no Zen1 nodes found, but the last-known master was a Zen1 node, so this is a rolling upgrade transportService.getThreadPool().generic().execute(() -> { try { - initialConfigurationConsumer.accept(new VotingConfiguration(discoveryNodes.stream() - .map(DiscoveryNode::getId).collect(Collectors.toSet()))); + Set nodeIds = new HashSet<>(); + discoveryNodes.forEach(n -> nodeIds.add(n.getId())); + + final Iterator knownNodeIdIterator = knownMasterNodeIds.iterator(); + while (nodeIds.size() < 2 * minimumMasterNodes - 1 && knownNodeIdIterator.hasNext()) { + nodeIds.add(knownNodeIdIterator.next()); + } + + final VotingConfiguration votingConfiguration = new VotingConfiguration(nodeIds); + assert votingConfiguration.hasQuorum( + discoveryNodes.stream().map(DiscoveryNode::getId).collect(Collectors.toList())); + assert 2 * minimumMasterNodes - 2 <= nodeIds.size() : nodeIds + " too small for " + minimumMasterNodes; + + initialConfigurationConsumer.accept(votingConfiguration); } catch (Exception e) { logger.debug("exception during bootstrapping upgrade, retrying", e); } finally { diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/Zen1IT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/Zen1IT.java index e8cd691129745..6ac753b5bc6d5 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/Zen1IT.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/Zen1IT.java @@ -22,9 +22,11 @@ import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest; import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsAction; import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsRequest; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.client.Client; import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -34,13 +36,19 @@ import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.discovery.Discovery; import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.gateway.MetaStateService; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalTestCluster.RestartCallback; import org.elasticsearch.test.discovery.TestZenDiscovery; +import org.elasticsearch.test.transport.MockTransportService; +import org.elasticsearch.transport.TransportService; +import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -48,10 +56,14 @@ import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING; import static org.elasticsearch.cluster.coordination.Coordinator.ZEN1_BWC_TERM; +import static org.elasticsearch.cluster.coordination.FollowersChecker.FOLLOWER_CHECK_ACTION_NAME; +import static org.elasticsearch.cluster.coordination.JoinHelper.START_JOIN_ACTION_NAME; +import static org.elasticsearch.cluster.coordination.PublicationTransportHandler.PUBLISH_STATE_ACTION_NAME; import static org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING; import static org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider.CLUSTER_ROUTING_EXCLUDE_GROUP_SETTING; import static org.elasticsearch.node.Node.NODE_NAME_SETTING; import static org.elasticsearch.test.InternalTestCluster.REMOVED_MINIMUM_MASTER_NODES; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -67,6 +79,10 @@ public class Zen1IT extends ESIntegTestCase { .put(TestZenDiscovery.USE_ZEN2.getKey(), true) .build(); + protected Collection> nodePlugins() { + return Collections.singletonList(MockTransportService.TestPlugin.class); + } + public void testZen2NodesJoiningZen1Cluster() { internalCluster().startNodes(randomIntBetween(1, 3), ZEN1_SETTINGS); internalCluster().startNodes(randomIntBetween(1, 3), ZEN2_SETTINGS); @@ -79,6 +95,56 @@ public void testZen1NodesJoiningZen2Cluster() { createIndex("test"); } + public void testMixedClusterDisruption() throws Exception { + final List nodes = internalCluster().startNodes(IntStream.range(0, 5) + .mapToObj(i -> i < 2 ? ZEN1_SETTINGS : ZEN2_SETTINGS).toArray(Settings[]::new)); + + final List transportServices = nodes.stream() + .map(n -> (MockTransportService) internalCluster().getInstance(TransportService.class, n)).collect(Collectors.toList()); + + logger.info("--> disrupting communications"); + + // The idea here is to make some of the Zen2 nodes believe the Zen1 nodes have gone away by introducing a network partition, so that + // they bootstrap themselves, but keep the Zen1 side of the cluster alive. + + // Set up a bridged network partition with the Zen1 nodes {0,1} on one side, Zen2 nodes {3,4} on the other, and node {2} in both + transportServices.get(0).addFailToSendNoConnectRule(transportServices.get(3)); + transportServices.get(0).addFailToSendNoConnectRule(transportServices.get(4)); + transportServices.get(1).addFailToSendNoConnectRule(transportServices.get(3)); + transportServices.get(1).addFailToSendNoConnectRule(transportServices.get(4)); + transportServices.get(3).addFailToSendNoConnectRule(transportServices.get(0)); + transportServices.get(3).addFailToSendNoConnectRule(transportServices.get(1)); + transportServices.get(4).addFailToSendNoConnectRule(transportServices.get(0)); + transportServices.get(4).addFailToSendNoConnectRule(transportServices.get(1)); + + // Nodes 3 and 4 will bootstrap, but we want to keep node 2 as part of the Zen1 cluster, so prevent any messages that might switch + // its allegiance + transportServices.get(3).addFailToSendNoConnectRule(transportServices.get(2), + PUBLISH_STATE_ACTION_NAME, FOLLOWER_CHECK_ACTION_NAME, START_JOIN_ACTION_NAME); + transportServices.get(4).addFailToSendNoConnectRule(transportServices.get(2), + PUBLISH_STATE_ACTION_NAME, FOLLOWER_CHECK_ACTION_NAME, START_JOIN_ACTION_NAME); + + logger.info("--> waiting for disconnected nodes to be removed"); + ensureStableCluster(3, nodes.get(0)); + + logger.info("--> creating index on Zen1 side"); + assertAcked(client(nodes.get(0)).admin().indices().create(new CreateIndexRequest("test")).get()); + assertFalse(client(nodes.get(0)).admin().cluster().health(new ClusterHealthRequest("test") + .waitForGreenStatus()).get().isTimedOut()); + + logger.info("--> waiting for disconnected nodes to bootstrap themselves"); + assertBusy(() -> assertTrue(IntStream.range(3, 5) + .mapToObj(n -> (Coordinator) internalCluster().getInstance(Discovery.class, nodes.get(n))) + .anyMatch(Coordinator::isInitialConfigurationSet))); + + logger.info("--> clearing disruption and waiting for cluster to reform"); + transportServices.forEach(MockTransportService::clearAllRules); + + ensureStableCluster(5, nodes.get(0)); + assertFalse(client(nodes.get(0)).admin().cluster().health(new ClusterHealthRequest("test") + .waitForGreenStatus()).get().isTimedOut()); + } + public void testMixedClusterFormation() throws Exception { final int zen1NodeCount = randomIntBetween(1, 3); final int zen2NodeCount = randomIntBetween(zen1NodeCount == 1 ? 2 : 1, 3); From 3dd4c96e8ea39908095c0812ea634f0e822cbfe9 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Thu, 31 Jan 2019 09:17:02 +0100 Subject: [PATCH 088/100] Update Lucene repo for 7.0.0-alpha2 (#37985) Let's help the users by giving them the right version to use for 7.0.0-alpha2 --- docs/java-rest/high-level/getting-started.asciidoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/java-rest/high-level/getting-started.asciidoc b/docs/java-rest/high-level/getting-started.asciidoc index f65a264cc33fc..685122771392c 100644 --- a/docs/java-rest/high-level/getting-started.asciidoc +++ b/docs/java-rest/high-level/getting-started.asciidoc @@ -83,7 +83,7 @@ dependencies { The very first releases of any major version (like a beta), might have been built on top of a Lucene Snapshot version. In such a case you will be unable to resolve the Lucene dependencies of the client. -For example, if you want to use the `7.0.0-alpha1` version which depends on Lucene `8.0.0-snapshot-6d9c714052`, you must +For example, if you want to use the `7.0.0-alpha2` version which depends on Lucene `8.0.0-snapshot-774e9aefbc`, you must define the following repository. For Maven: @@ -93,7 +93,7 @@ For Maven: elastic-lucene-snapshots Elastic Lucene Snapshots - http://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/6d9c714052 + http://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/774e9aefbc true false @@ -104,7 +104,7 @@ For Gradle: ["source","groovy",subs="attributes"] -------------------------------------------------- maven { - url 'http://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/6d9c714052' + url 'http://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/774e9aefbc' } -------------------------------------------------- From 3332e332c729c0918f70561d70345703aa79eede Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 31 Jan 2019 09:51:03 +0100 Subject: [PATCH 089/100] Fix typo in docs. (#38018) This has been introduced in #37871. --- docs/reference/mapping/removal_of_types.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/mapping/removal_of_types.asciidoc b/docs/reference/mapping/removal_of_types.asciidoc index b9066a4c7af49..5b7138ffa3f78 100644 --- a/docs/reference/mapping/removal_of_types.asciidoc +++ b/docs/reference/mapping/removal_of_types.asciidoc @@ -606,5 +606,5 @@ PUT index-2-01 In case of implicit index creation, because of documents that get indexed in an index that doesn't exist yet, the template is always honored. This is -usually not a problem due to the fact that typless index calls work on typed +usually not a problem due to the fact that typeless index calls work on typed indices. From 8309e0ce77321a919b13255f128203246cf5945f Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 31 Jan 2019 09:52:25 +0100 Subject: [PATCH 090/100] Minor fixes in the release notes script. (#37967) The `:beats` label is actually `:Beats` in Github. `:Core/Build` is now `:Core/Infra/Build`. --- dev-tools/es_release_notes.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-tools/es_release_notes.pl b/dev-tools/es_release_notes.pl index cf19a1cd9ddf5..e911b5a5a4a4c 100755 --- a/dev-tools/es_release_notes.pl +++ b/dev-tools/es_release_notes.pl @@ -32,7 +32,7 @@ ">enhancement", ">bug", ">regression", ">upgrade" ); my %Ignore = map { $_ => 1 } - ( ">non-issue", ">refactoring", ">docs", ">test", ">test-failure", ":Core/Build", "backport" ); + ( ">non-issue", ">refactoring", ">docs", ">test", ">test-failure", ":Core/Infra/Build", "backport" ); my %Group_Labels = ( '>breaking' => 'Breaking changes', @@ -48,7 +48,7 @@ my %Area_Overrides = ( ':ml' => 'Machine Learning', - ':beats' => 'Beats Plugin', + ':Beats' => 'Beats Plugin', ':Docs' => 'Docs Infrastructure' ); From 22d32900784600f99b907ef3e81aca94d5c78f9c Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Thu, 31 Jan 2019 10:52:49 +0200 Subject: [PATCH 091/100] SQL: Added SSL configuration options tests (#37875) * Added SSL configuration options tests Removed the allow.self.signed option from the documentation since we allow by default self signed certificates as well. * Added more tests --- docs/reference/sql/endpoints/jdbc.asciidoc | 2 - .../sql/jdbc/JdbcConfigurationTests.java | 157 +++++++++++++++++- .../xpack/sql/client/SslConfig.java | 2 +- 3 files changed, 157 insertions(+), 4 deletions(-) diff --git a/docs/reference/sql/endpoints/jdbc.asciidoc b/docs/reference/sql/endpoints/jdbc.asciidoc index e9af3492adcaa..56c68fd34937f 100644 --- a/docs/reference/sql/endpoints/jdbc.asciidoc +++ b/docs/reference/sql/endpoints/jdbc.asciidoc @@ -115,8 +115,6 @@ Query timeout (in seconds). That is the maximum amount of time waiting for a que `ssl.truststore.pass`:: trust store password -`ssl.cert.allow.self.signed` (default `false`):: Whether or not to allow self signed certificates - `ssl.protocol`(default `TLS`):: SSL protocol to be used [float] diff --git a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfigurationTests.java b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfigurationTests.java index 5f0f523fb009f..dac9dbba61776 100644 --- a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfigurationTests.java +++ b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfigurationTests.java @@ -6,9 +6,16 @@ package org.elasticsearch.xpack.sql.jdbc; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.sql.client.SslConfig; +import java.net.URI; +import java.net.URISyntaxException; +import java.sql.DriverManager; import java.sql.SQLException; +import java.util.HashMap; +import java.util.Map; import java.util.Properties; +import java.util.stream.Collectors; import static org.elasticsearch.xpack.sql.client.ConnectionConfiguration.CONNECT_TIMEOUT; import static org.elasticsearch.xpack.sql.client.ConnectionConfiguration.PAGE_TIMEOUT; @@ -130,5 +137,153 @@ public void testTimoutOverride() throws Exception { assertThat(ci.pageTimeout(), equalTo(4L)); } - + public void testSSLPropertiesInUrl() throws Exception { + Map urlPropMap = sslProperties(); + + Properties allProps = new Properties(); + allProps.putAll(urlPropMap); + String sslUrlProps = urlPropMap.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining("&")); + + assertSslConfig(allProps, ci("jdbc:es://test?" + sslUrlProps.toString()).sslConfig()); + } + + public void testSSLPropertiesInUrlAndProperties() throws Exception { + Map urlPropMap = new HashMap<>(4); + urlPropMap.put("ssl", "false"); + urlPropMap.put("ssl.protocol", "SSLv3"); + urlPropMap.put("ssl.keystore.location", "/abc/xyz"); + urlPropMap.put("ssl.keystore.pass", "mypass"); + + Map propMap = new HashMap<>(4); + propMap.put("ssl.keystore.type", "PKCS12"); + propMap.put("ssl.truststore.location", "/foo/bar"); + propMap.put("ssl.truststore.pass", "anotherpass"); + propMap.put("ssl.truststore.type", "jks"); + + Properties props = new Properties(); + props.putAll(propMap); + String sslUrlProps = urlPropMap.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining("&")); + + Properties allProps = new Properties(); + allProps.putAll(urlPropMap); + allProps.putAll(propMap); + assertSslConfig(allProps, JdbcConfiguration.create("jdbc:es://test?" + sslUrlProps.toString(), props, 0).sslConfig()); + } + + public void testSSLPropertiesOverride() throws Exception { + Map urlPropMap = sslProperties(); + Map propMap = new HashMap<>(8); + propMap.put("ssl", "false"); + propMap.put("ssl.protocol", "TLS"); + propMap.put("ssl.keystore.location", "/xyz"); + propMap.put("ssl.keystore.pass", "different_mypass"); + propMap.put("ssl.keystore.type", "JKS"); + propMap.put("ssl.truststore.location", "/baz"); + propMap.put("ssl.truststore.pass", "different_anotherpass"); + propMap.put("ssl.truststore.type", "PKCS11"); + + Properties props = new Properties(); + props.putAll(propMap); + String sslUrlProps = urlPropMap.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining("&")); + assertSslConfig(props, JdbcConfiguration.create("jdbc:es://test?" + sslUrlProps.toString(), props, 0).sslConfig()); + } + + public void testDriverConfigurationWithSSLInURL() { + Map urlPropMap = sslProperties(); + + Properties allProps = new Properties(); + allProps.putAll(urlPropMap); + String sslUrlProps = urlPropMap.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining("&")); + + try { + DriverManager.getDriver("jdbc:es://test?" + sslUrlProps); + } catch (SQLException sqle) { + fail("Driver registration should have been successful. Error: " + sqle); + } + } + + public void testDataSourceConfigurationWithSSLInURL() throws SQLException, URISyntaxException { + Map urlPropMap = sslProperties(); + + Properties allProps = new Properties(); + allProps.putAll(urlPropMap); + String sslUrlProps = urlPropMap.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining("&")); + + EsDataSource dataSource = new EsDataSource(); + String address = "jdbc:es://test?" + sslUrlProps; + dataSource.setUrl(address); + JdbcConnection connection = null; + + try { + connection = (JdbcConnection) dataSource.getConnection(); + } catch (SQLException sqle) { + fail("Connection creation should have been successful. Error: " + sqle); + } + + assertEquals(address, connection.getURL()); + assertSslConfig(allProps, connection.cfg.sslConfig()); + } + + public void testTyposInSslConfigInUrl(){ + assertJdbcSqlExceptionFromUrl("ssl.protocl", "ssl.protocol"); + assertJdbcSqlExceptionFromUrl("sssl", "ssl"); + assertJdbcSqlExceptionFromUrl("ssl.keystore.lction", "ssl.keystore.location"); + assertJdbcSqlExceptionFromUrl("ssl.keystore.pss", "ssl.keystore.pass"); + assertJdbcSqlExceptionFromUrl("ssl.keystore.typ", "ssl.keystore.type"); + assertJdbcSqlExceptionFromUrl("ssl.trustsore.location", "ssl.truststore.location"); + assertJdbcSqlExceptionFromUrl("ssl.tuststore.pass", "ssl.truststore.pass"); + assertJdbcSqlExceptionFromUrl("ssl.ruststore.type", "ssl.truststore.type"); + } + + public void testTyposInSslConfigInProperties() { + assertJdbcSqlExceptionFromProperties("ssl.protocl", "ssl.protocol"); + assertJdbcSqlExceptionFromProperties("sssl", "ssl"); + assertJdbcSqlExceptionFromProperties("ssl.keystore.lction", "ssl.keystore.location"); + assertJdbcSqlExceptionFromProperties("ssl.keystore.pss", "ssl.keystore.pass"); + assertJdbcSqlExceptionFromProperties("ssl.keystore.typ", "ssl.keystore.type"); + assertJdbcSqlExceptionFromProperties("ssl.trustsore.location", "ssl.truststore.location"); + assertJdbcSqlExceptionFromProperties("ssl.tuststore.pass", "ssl.truststore.pass"); + assertJdbcSqlExceptionFromProperties("ssl.ruststore.type", "ssl.truststore.type"); + } + + private Map sslProperties() { + Map sslPropertiesMap = new HashMap<>(8); + // always using "false" so that the SSLContext doesn't actually start verifying the keystore and trustore + // locations, as we don't have file permissions to access them. + sslPropertiesMap.put("ssl", "false"); + sslPropertiesMap.put("ssl.protocol", "SSLv3"); + sslPropertiesMap.put("ssl.keystore.location", "/abc/xyz"); + sslPropertiesMap.put("ssl.keystore.pass", "mypass"); + sslPropertiesMap.put("ssl.keystore.type", "PKCS12"); + sslPropertiesMap.put("ssl.truststore.location", "/foo/bar"); + sslPropertiesMap.put("ssl.truststore.pass", "anotherpass"); + sslPropertiesMap.put("ssl.truststore.type", "jks"); + + return sslPropertiesMap; + } + + private void assertSslConfig(Properties allProperties, SslConfig sslConfig) throws URISyntaxException { + // because SslConfig doesn't expose its internal properties (and it shouldn't), + // we compare a newly created SslConfig with the one from the JdbcConfiguration with the equals() method + SslConfig mockSslConfig = new SslConfig(allProperties, new URI("http://test:9200/")); + assertEquals(mockSslConfig, sslConfig); + } + + private void assertJdbcSqlExceptionFromUrl(String wrongSetting, String correctSetting) { + String url = "jdbc:es://test?" + wrongSetting + "=foo"; + assertJdbcSqlException(wrongSetting, correctSetting, url, null); + } + + private void assertJdbcSqlExceptionFromProperties(String wrongSetting, String correctSetting) { + String url = "jdbc:es://test"; + Properties props = new Properties(); + props.put(wrongSetting, correctSetting); + assertJdbcSqlException(wrongSetting, correctSetting, url, props); + } + + private void assertJdbcSqlException(String wrongSetting, String correctSetting, String url, Properties props) { + JdbcSQLException ex = expectThrows(JdbcSQLException.class, + () -> JdbcConfiguration.create(url, props, 0)); + assertEquals("Unknown parameter [" + wrongSetting + "] ; did you mean [" + correctSetting + "]", ex.getMessage()); + } } diff --git a/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/SslConfig.java b/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/SslConfig.java index 1b19c385db4d1..63e07dc8b169b 100644 --- a/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/SslConfig.java +++ b/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/SslConfig.java @@ -63,7 +63,7 @@ public class SslConfig { private final SSLContext sslContext; - SslConfig(Properties settings, URI baseURI) { + public SslConfig(Properties settings, URI baseURI) { boolean isSchemaPresent = baseURI.getScheme() != null; boolean isSSLPropertyPresent = settings.getProperty(SSL) != null; boolean isHttpsScheme = "https".equals(baseURI.getScheme()); From 62b1874b927c15c239d7330baa841440373d6e62 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Thu, 31 Jan 2019 10:36:12 +0100 Subject: [PATCH 092/100] Disable BWC tests during backport (#38074) This pull request disables BWC tests while backporting #37899 to 6.x. --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index e5bc1ab3ba986..78beebb157a34 100644 --- a/build.gradle +++ b/build.gradle @@ -159,8 +159,8 @@ task verifyVersions { * the enabled state of every bwc task. It should be set back to true * after the backport of the backcompat code is complete. */ -final boolean bwc_tests_enabled = true -final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */ +final boolean bwc_tests_enabled = false +final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/37899" /* place a PR link here when committing bwc changes */ if (bwc_tests_enabled == false) { if (bwc_tests_disabled_issue.isEmpty()) { throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false") From cde126dbff1e443bdaa13b5d1b541dfb037db028 Mon Sep 17 00:00:00 2001 From: Tim Vernum Date: Thu, 31 Jan 2019 20:59:50 +1100 Subject: [PATCH 093/100] Enable SSL in reindex with security QA tests (#37600) Update the x-pack/qa/reindex-tests-with-security integration tests to run with TLS enabled on the Rest interface. Relates: #37527 --- .../test/rest/ESRestTestCase.java | 3 +- .../reindex-tests-with-security/build.gradle | 77 ++++++++++++++++-- ...ndexWithSecurityClientYamlTestSuiteIT.java | 31 ++++++- .../test/15_reindex_from_remote.yml | 18 ++-- .../src/test/resources/ssl/README.asciidoc | 28 +++++++ .../src/test/resources/ssl/ca.crt | 20 +++++ .../src/test/resources/ssl/ca.key | 30 +++++++ .../src/test/resources/ssl/ca.p12 | Bin 0 -> 1130 bytes .../src/test/resources/ssl/http.crt | 22 +++++ .../src/test/resources/ssl/http.key | 30 +++++++ 10 files changed, 240 insertions(+), 19 deletions(-) create mode 100644 x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/README.asciidoc create mode 100644 x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/ca.crt create mode 100644 x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/ca.key create mode 100644 x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/ca.p12 create mode 100644 x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/http.crt create mode 100644 x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/http.key diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 23e78d5492ff7..c363b7f4f6c92 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -708,7 +708,8 @@ protected static void configureClient(RestClientBuilder builder, Settings settin throw new IllegalStateException(TRUSTSTORE_PATH + " is set but points to a non-existing file"); } try { - KeyStore keyStore = KeyStore.getInstance("jks"); + final String keyStoreType = keystorePath.endsWith(".p12") ? "PKCS12" : "jks"; + KeyStore keyStore = KeyStore.getInstance(keyStoreType); try (InputStream is = Files.newInputStream(path)) { keyStore.load(is, keystorePass.toCharArray()); } diff --git a/x-pack/qa/reindex-tests-with-security/build.gradle b/x-pack/qa/reindex-tests-with-security/build.gradle index 0bd51f483eaad..3d415e0e2922a 100644 --- a/x-pack/qa/reindex-tests-with-security/build.gradle +++ b/x-pack/qa/reindex-tests-with-security/build.gradle @@ -1,3 +1,11 @@ +import javax.net.ssl.HttpsURLConnection +import javax.net.ssl.KeyManager +import javax.net.ssl.SSLContext +import javax.net.ssl.TrustManagerFactory +import java.nio.charset.StandardCharsets +import java.security.KeyStore +import java.security.SecureRandom + apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' @@ -9,13 +17,31 @@ dependencies { testCompile project(path: ':modules:reindex') } +forbiddenPatterns { + exclude '**/*.key' + exclude '**/*.pem' + exclude '**/*.p12' + exclude '**/*.jks' +} + +File caFile = project.file('src/test/resources/ssl/ca.p12') + integTestCluster { // Whitelist reindexing from the local node so we can test it. + extraConfigFile 'http.key', project.projectDir.toPath().resolve('src/test/resources/ssl/http.key') + extraConfigFile 'http.crt', project.projectDir.toPath().resolve('src/test/resources/ssl/http.crt') + extraConfigFile 'ca.p12', caFile setting 'reindex.remote.whitelist', '127.0.0.1:*' setting 'xpack.ilm.enabled', 'false' setting 'xpack.security.enabled', 'true' setting 'xpack.ml.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' + setting 'xpack.security.http.ssl.enabled', 'true' + setting 'xpack.security.http.ssl.certificate', 'http.crt' + setting 'xpack.security.http.ssl.key', 'http.key' + setting 'xpack.security.http.ssl.key_passphrase', 'http-password' + setting 'reindex.ssl.truststore.path', 'ca.p12' + setting 'reindex.ssl.truststore.password', 'password' extraConfigFile 'roles.yml', 'roles.yml' [ test_admin: 'superuser', @@ -31,13 +57,48 @@ integTestCluster { 'bin/elasticsearch-users', 'useradd', user, '-p', 'x-pack-test-password', '-r', role } waitCondition = { node, ant -> - File tmpFile = new File(node.cwd, 'wait.success') - ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow", - dest: tmpFile.toString(), - username: 'test_admin', - password: 'x-pack-test-password', - ignoreerrors: true, - retries: 10) - return tmpFile.exists() + // Load the CA PKCS#12 file as a truststore + KeyStore ks = KeyStore.getInstance("PKCS12"); + ks.load(caFile.newInputStream(), 'password'.toCharArray()); + TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + tmf.init(ks); + + // Configre a SSL context for TLS1.2 using our CA trust manager + SSLContext sslContext = SSLContext.getInstance("TLSv1.2"); + sslContext.init(new KeyManager[0], tmf.getTrustManagers(), new SecureRandom()); + + // Check whether the cluster has started + URL url = new URL("https://${node.httpUri()}/_cluster/health?wait_for_nodes=${numNodes}&wait_for_status=yellow"); + for (int i = 20; i >= 0; i--) { + // we use custom wait logic here for HTTPS + HttpsURLConnection httpURLConnection = null; + try { + logger.info("Trying ${url}"); + httpURLConnection = (HttpsURLConnection) url.openConnection(); + httpURLConnection.setSSLSocketFactory(sslContext.getSocketFactory()); + httpURLConnection.setRequestProperty("Authorization", + "Basic " + Base64.getEncoder().encodeToString("test_admin:x-pack-test-password".getBytes(StandardCharsets.UTF_8))); + httpURLConnection.setRequestMethod("GET"); + httpURLConnection.connect(); + if (httpURLConnection.getResponseCode() == 200) { + logger.info("Cluster has started"); + return true; + } else { + logger.debug("HTTP response was [{}]", httpURLConnection.getResponseCode()); + } + } catch (IOException e) { + if (i == 0) { + logger.error("Failed to call cluster health - " + e) + } + logger.debug("Call to [{}] threw an exception", url, e) + } finally { + if (httpURLConnection != null) { + httpURLConnection.disconnect(); + } + } + // did not start, so wait a bit before trying again + Thread.sleep(750L); + } + return false; } } diff --git a/x-pack/qa/reindex-tests-with-security/src/test/java/org/elasticsearch/xpack/security/ReindexWithSecurityClientYamlTestSuiteIT.java b/x-pack/qa/reindex-tests-with-security/src/test/java/org/elasticsearch/xpack/security/ReindexWithSecurityClientYamlTestSuiteIT.java index 67ebf16f426ed..76715613e3c36 100644 --- a/x-pack/qa/reindex-tests-with-security/src/test/java/org/elasticsearch/xpack/security/ReindexWithSecurityClientYamlTestSuiteIT.java +++ b/x-pack/qa/reindex-tests-with-security/src/test/java/org/elasticsearch/xpack/security/ReindexWithSecurityClientYamlTestSuiteIT.java @@ -7,12 +7,18 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - +import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.FileNotFoundException; +import java.net.URL; +import java.nio.file.Path; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; @@ -20,6 +26,8 @@ public class ReindexWithSecurityClientYamlTestSuiteIT extends ESClientYamlSuiteT private static final String USER = "test_admin"; private static final String PASS = "x-pack-test-password"; + private static Path httpTrustStore; + public ReindexWithSecurityClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); } @@ -29,6 +37,25 @@ public static Iterable parameters() throws Exception { return ESClientYamlSuiteTestCase.createParameters(); } + @BeforeClass + public static void findTrustStore( ) throws Exception { + final URL resource = ReindexWithSecurityClientYamlTestSuiteIT.class.getResource("/ssl/ca.p12"); + if (resource == null) { + throw new FileNotFoundException("Cannot find classpath resource /ssl/ca.p12"); + } + httpTrustStore = PathUtils.get(resource.toURI()); + } + + @AfterClass + public static void cleanupStatics() { + httpTrustStore = null; + } + + @Override + protected String getProtocol() { + return "https"; + } + /** * All tests run as a an administrative user but use es-security-runas-user to become a less privileged user. */ @@ -37,6 +64,8 @@ protected Settings restClientSettings() { String token = basicAuthHeaderValue(USER, new SecureString(PASS.toCharArray())); return Settings.builder() .put(ThreadContext.PREFIX + ".Authorization", token) + .put(TRUSTSTORE_PATH , httpTrustStore) + .put(TRUSTSTORE_PASSWORD, "password") .build(); } } diff --git a/x-pack/qa/reindex-tests-with-security/src/test/resources/rest-api-spec/test/15_reindex_from_remote.yml b/x-pack/qa/reindex-tests-with-security/src/test/resources/rest-api-spec/test/15_reindex_from_remote.yml index a68b5262c020e..e41fba0d7a55e 100644 --- a/x-pack/qa/reindex-tests-with-security/src/test/resources/rest-api-spec/test/15_reindex_from_remote.yml +++ b/x-pack/qa/reindex-tests-with-security/src/test/resources/rest-api-spec/test/15_reindex_from_remote.yml @@ -26,7 +26,7 @@ body: source: remote: - host: http://${host} + host: https://${host} username: test_admin password: x-pack-test-password index: source @@ -63,7 +63,7 @@ body: source: remote: - host: http://${host} + host: https://${host} username: minimal_user password: x-pack-test-password index: source @@ -110,7 +110,7 @@ body: source: remote: - host: http://${host} + host: https://${host} username: readonly_user password: x-pack-test-password index: source @@ -156,7 +156,7 @@ body: source: remote: - host: http://${host} + host: https://${host} username: dest_only_user password: x-pack-test-password index: source @@ -198,7 +198,7 @@ body: source: remote: - host: http://${host} + host: https://${host} username: test_admin password: x-pack-test-password index: source @@ -259,7 +259,7 @@ body: source: remote: - host: http://${host} + host: https://${host} username: can_not_see_hidden_docs_user password: x-pack-test-password index: source @@ -318,7 +318,7 @@ body: source: remote: - host: http://${host} + host: https://${host} username: can_not_see_hidden_fields_user password: x-pack-test-password index: source @@ -386,7 +386,7 @@ body: source: remote: - host: http://${host} + host: https://${host} username: test_admin password: badpass index: source @@ -422,7 +422,7 @@ body: source: remote: - host: http://${host} + host: https://${host} index: source dest: index: dest diff --git a/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/README.asciidoc b/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/README.asciidoc new file mode 100644 index 0000000000000..363f39ba012fd --- /dev/null +++ b/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/README.asciidoc @@ -0,0 +1,28 @@ += Keystore Details +This document details the steps used to create the certificate and keystore files in this directory. + +== Instructions on generating certificates +The certificates in this directory have been generated using elasticsearch-certutil (7.0.0 SNAPSHOT) + +[source,shell] +----------------------------------------------------------------------------------------------------------- +elasticsearch-certutil ca --pem --out=ca.zip --pass="ca-password" --days=3500 +unzip ca.zip +mv ca/ca.* ./ +----------------------------------------------------------------------------------------------------------- + +[source,shell] +----------------------------------------------------------------------------------------------------------- +elasticsearch-certutil cert --pem --name=http --out=http.zip --pass="http-password" --days=3500 \ + --ca-cert=ca.crt --ca-key=ca.key --ca-pass="ca-password" \ + --dns=localhost --dns=localhost.localdomain --dns=localhost4 --dns=localhost4.localdomain4 --dns=localhost6 --dns=localhost6.localdomain6 \ + --ip=127.0.0.1 --ip=0:0:0:0:0:0:0:1 + +unzip http.zip +mv http/http.* ./ +----------------------------------------------------------------------------------------------------------- + +[source,shell] +----------------------------------------------------------------------------------------------------------- +keytool -importcert -file ca.crt -keystore ca.p12 -storetype PKCS12 -storepass "password" -alias ca +----------------------------------------------------------------------------------------------------------- diff --git a/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/ca.crt b/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/ca.crt new file mode 100644 index 0000000000000..dcbe636e2a594 --- /dev/null +++ b/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/ca.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDSjCCAjKgAwIBAgIVAMp4ojQbvgxx3HBRFHadTvCjFn1+MA0GCSqGSIb3DQEB +CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu +ZXJhdGVkIENBMB4XDTE4MTIxMzA2MDU0OVoXDTI4MDcxMzA2MDU0OVowNDEyMDAG +A1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5lcmF0ZWQgQ0Ew +ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCDtMpBiqR2EaLHC8jNojf9 +G4xlqFYj+pLzQidPHQlmqEDYYpSUmSVxh2GT7f6VQ7acdlFecSIfbvngGE94aBFB +sSQwzrjk0Bgq3+31nQDdM9DwHPQxYWdq20mxs0qztfpV0BfzsS4hdTHVK3ZvtaN8 +D+FTTvugM/e/PZxEXa2yFVt7GfCe2mF6DLvJpm86Eeyfr9HPZc6QK2vKaNkeaFSr +WFyovb8ivLb6yGMQva/fnQRAJNLZi0YnsMwUhn/Xe1MyfeRyLmkLvF+Q3XwiYInt +0721DMUH4VYaQ2EV76g3v0mxvbCdHMCRVudvlqiO3y4AXyq9RDJ5f3AZIEX8aBAr +AgMBAAGjUzBRMB0GA1UdDgQWBBTln4o7tJW/VyYSNJXbgrYYnIR3czAfBgNVHSME +GDAWgBTln4o7tJW/VyYSNJXbgrYYnIR3czAPBgNVHRMBAf8EBTADAQH/MA0GCSqG +SIb3DQEBCwUAA4IBAQB8AzUs0DIVmuasZoN5ftBOzNB2XUHI3p95Yju3lF+9E0i4 +ZyAjqcQoNaDSHrd9bzhmQuLiPmN+dPEwGNhlg5ddclthfwY4qy+IxoIUM6L/vFlF +ApPx+XZK3zZtv/kXqjz8ZWA8Qj4BVWOo3XK4HodJkoMDIkhWPQXlA8BEJDNUnirl +8HTlibnihKvzGmZHEWvgm6YrUyS4YknUvafROW/EUm4Gl4zniFuLG8VVN/2dbJmy +v2xMsqji8Pf+2ZnZ/aXS0bg4hzGyPBljoifEI7lj0twg5zpEXeCZJ7BgsoFicgve +iYZV9yrDBHognEbFAIywiK3+GXrqAkvB/OQiGCTM +-----END CERTIFICATE----- diff --git a/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/ca.key b/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/ca.key new file mode 100644 index 0000000000000..048993cae880a --- /dev/null +++ b/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/ca.key @@ -0,0 +1,30 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,2DF8E98BAAF74EB5 + +Y5r4e8s5XV7aUr12V0PSfr1+67J+Ao6fAG0fjxM0M7Qv3IIghS9OdW0YKoWc8h8e +tlJbSrEAnpvopROqCXh860XCdGrDwwTKVEnazgvXb64+CIcVuZXK/HesrBiQIUwz +O/rZKN8HKtM5KTfVqpCtNsCu6TDenkHUEz5uOaG3p3/rvhFtsGp/4PGt5slYt/Z8 +g5J0EJeLwRgJaRVR70/3LhmUryZxM4TPMvHjCU7GI4YbXzFzp8qbRSujWr4/l5qm +4Piid3pyxN1L47TviB6jRWt7XZrOcvr7Glqjuz0ak9beUyidL3QUJAgZGQD9O6zj +iPaGI/9AF01fAo9J8N7LDmGacPz9dvpvIsXOXfz+7COtXhKki2VqEx/XwWHz2opw +82uMj59bSBrCzf+Y417G60Me/mPuYdxiqRoFKsszrsH7HiQgQroBM/X8Trq6OmXc +CGDsYO0tUT0xYVFoW1j3rMGh4wV9z5G3LSKFtO54uHdGUmJUSFATcwOnME9acUUj +jG9qCn/dIkXjKIZ/jwaaA65GG/P60VGOJG+AjHbiBbEPXD/IA++Y4X2M2H4jvQrr +oG7bLD4Zaa/B88Jv7ymZh88SCZpYqd0I96G5DSzlzoNpqLwhNmcdy+ViSIqlFfD9 +HpbQwT0mQJeUPj8KmXtOl2GVunwNkdBEaRURXiD4l9CPCmFXGb1RKt02RY6Nvf6X +w9/SvipGsCaGbALoQb1UvKiL7JqU9eYoslYb84A+abbPQtiy7MBZqbyhNQ2PI2ct +FV1z+h5GV/wzI1y+CWeCJWhjysShMBNv/eOfp8iStkIqI7M+2qKHyzMusqZxov3Y +8QgcQqbDSR/mWZ4Kl1/h/RC+qPy20bgeYAT5VvXhBasu7Mzq+5qiZ+T9FK/nTkq6 +xLMYGLbFe2tRWJMBxeHVu/YuG8gwjWVrhalfFmWeh2skqPIeymGpTxU42XUaI4zr +7CVoyWalnMYZWbGculaVFutSyIlqshY0w56PXVpt5usow968rTw+Nf8YeQ/pLFi4 +r0fteQSOEXdwGgy8/fcvhzaPbgJfTcIbaRgP89q/HORYDjm/P03jHXmiT11ZeF84 +pqtGRTJqCbL7n/vc/5gXdvYt88alxEn9sIyhNugpXWp9EJefnyUscxI036wbBK4O +sNSewqIpp+kGn/Xf/PqfkKQVZkA9YacMcPiKoGVYExoujukfeHwZ/jq7geOqYa+H ++NPUd5VS8lxX/lhAt3Nit97UnJ2oQvbHsV/+eJ65/1e41hS0h1xpzd4HLhDoQEfV +Q0L+1h9cbwU/IyUXK+4fr4nUNolSYNzXfurGKDLVtjFpR+naupr2CwQU7gKHKikF +7GuogsTbtK9L3jkIla/lYTqKiJlz/vA6erTmI06aENt0DnnVKPaQZhJ8571lKmRV +xe+e56R4s0AZBOpZjykkr7hDWQ2QGwbgKOYHF9KRl/yQZwD1ezRu3feSUdPkRrLY +efPH24L0jEampqIhx4XGFbhYX/WnuvneA2oiswmB4zR4YT2F8PeMw4gd3t0nGljz +U/NbbQ7P5ZP8JjQbHecSIZf262mHCGuWtnul9T4DjTubyD3LO6AXxw== +-----END RSA PRIVATE KEY----- diff --git a/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/ca.p12 b/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/ca.p12 new file mode 100644 index 0000000000000000000000000000000000000000..78e8e5a97ef4e398e10e90c8666ca3d76cad3014 GIT binary patch literal 1130 zcmV-w1eN$$ zLrxrqyGpH=-*iD~x{F&1K-)Xcxdk$(hU~zJ(2WT#YVrg42ISA^#pEWx6k+x_GM4bf zw)6o-vu>54r8YQB;J?O0_jZJ-ff)jkw6>=v<2-AntN0SWFIoa#Dk$xLt<92u8u-A6 zehO8Y;w1>cCJJG&<&~*7ST%V@ENr-u@x)YXG{LMVy0Z@L(dgzwa5*Ir-Eja0&T@7+ zq@02!fEqvjQzbtX#pFuK%rv)?y@zj&A)WD!=8S}xbO!~yG0d()`v@`5Nop&`e+EOM z0caspBLzWH9XlAELjxHF|E`HOYh@Giz3~=Ow$@aj(mJg^U5NQ_U0ge3rIdun-CnGGHKoI@>$M*pOW zJ)|H$m5*$_@s1WubTYZa4gRc8^v^A_?$lwcEU#*rE#2V4kEDd64^^f6`&VtR4SIpZ zgY+}{aPM~l8{>~ojfF1?5s+5m_dy3nSIAx5w2(Hg?Eci?FXLMo9$8xMUMP-`mrVSa zgb{$e8AB^oSxB<71yaRb;kpCp2FCX z;jcK07d)VB_@50wa#r!r)k~fr$VomJMW=1np=X66I2#70^_+~RqMifKrj%aO&a%Xb zxeot)Y(H;E?fG36tFc(kRM;r3rA%2|!6ii% z!#~%j&awt`4V}A6NCmQiC?;YwB4le;fpO!Y=Y4+}1?AYmE7622!t0|&Z@UZWh!etS zE$yYKU;xGH@%mNUxFKb9mqZc6UrQR{F&Ez_X43rP0OpWqU_FoA9j^!izuVvMPu0>; z9(E-yOa&c)yT$c3h5oJl3|nl%v(-@-1ec+h`koi9?bAQ`|`3t!PW{$bTmchRxB>Vu;=Gwh{;Km}^$K?qsDXjK4jePx;CSQnVu z@yKRziswH}*(6D8{n>iPs=)yGB*m#g7T*4(gFS0(YtmeD_Y$}m1KAutIB1uG5%0vZJX1QhI{ wI^l>bEBn06A+0s{etppvExFaQ7m literal 0 HcmV?d00001 diff --git a/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/http.crt b/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/http.crt new file mode 100644 index 0000000000000..a41d892f100dc --- /dev/null +++ b/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/http.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDszCCApugAwIBAgIVAImurbHhcSbc4LTPdTawV03y+KXtMA0GCSqGSIb3DQEB +CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu +ZXJhdGVkIENBMB4XDTE4MTIxNDA1MDcwNloXDTI4MDcxNDA1MDcwNlowDzENMAsG +A1UEAxMEaHR0cDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAI5zBwHm +5kwiV3tYlOl0AEruYNUjRfYwQ2OtBIILCgnu0USMx4r1I6IoHDuLl9Z197I1UfSF +c49hG5U82gAFtWblYoITPkzW50sSB7un/rehXkwTIMbl0i024rWQfGGj6uGHmlU4 +T+2YWZNksdGEWx7pcG9WZ4r7rjCy7A0SbewhHmD9SxxZgfsW2UI1bu/iXKC8cb7L +DYBnYCDiYheAA3zOPm1zIB12BDsMuFBF4vIEHlxwOH9pH8jC6vuSEnMqTct81uN9 +6EwhPvEixrklffj3XDDYYQzoyF3yiabBt3PTm5v56IadcxjQZT/S5fGBuApTLfdV +w6aCzxTa0vEx0k0CAwEAAaOB4DCB3TAdBgNVHQ4EFgQUwQ35Nzes5weOuudemw3z +MK9ZlkcwHwYDVR0jBBgwFoAU5Z+KO7SVv1cmEjSV24K2GJyEd3MwgY8GA1UdEQSB +hzCBhIIJbG9jYWxob3N0ghdsb2NhbGhvc3Q2LmxvY2FsZG9tYWluNocEfwAAAYcQ +AAAAAAAAAAAAAAAAAAAAAYIKbG9jYWxob3N0NIIKbG9jYWxob3N0NoIVbG9jYWxo +b3N0LmxvY2FsZG9tYWlughdsb2NhbGhvc3Q0LmxvY2FsZG9tYWluNDAJBgNVHRME +AjAAMA0GCSqGSIb3DQEBCwUAA4IBAQBQ7B6jdsMjT7qHQJV68kVdrJwDLekpWPvQ +f+YgPZaWkQoVI7rpBJGm7ZY49RI61JLA1SDxjHS3wL3EYRo1FuXwQj6K/h9wxrpn +is1Ib9IewxeueGhi0DMr+Wf5Nh6cDC7I0Uftr2NJsmwivZV9ZlECjckpIZjwIHpb +imtb27MBcVzWjVLL+NLDa2upuVVYdeiuIcbpMqjOFW7mn6/FczgbMjg8zOQG7+fF +pUEduOJDkmxLe14aKagqyaZDMpX1g1CiM3V899/kYXMPPP9F5f7WOg0+QGfBSig3 +3KNPfiQyaIeIePhtRC4iPgP1WI5QaiOVd0GwNwp1W39GeJv6/Wit +-----END CERTIFICATE----- diff --git a/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/http.key b/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/http.key new file mode 100644 index 0000000000000..5a789a2e1553e --- /dev/null +++ b/x-pack/qa/reindex-tests-with-security/src/test/resources/ssl/http.key @@ -0,0 +1,30 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,73350501C44BDD9D + +3LAGS/e2rkuTBIyIVhFEFy/uG4gUJlJ+5rBTYkmxIfZWkUin+yEoThvJsYptiXCr +hjDIXhZkob2NVfEae9KRW6rETs/7ZwZfR5XpgeYbre8/JyfEo0I5QvcatZMAJ6Gz +3UEXnjIxrowg3voRW12oa6ppa8K1zwfr5y7GscCuJvL/eBp+/TkFAtCbcLwnvUBI +rZ3VjsUO5VvElRgmttI8L9zyt6KJA2laM6D/DZW25JYa++BNkFMBxP7DiEl+fmGL +x1d+ozHVxcxVANO/UrtkvBqyF5M9S69vbz0XSCZX/oV/a0IuzzHiZy0YnK2TPKZw +V6IgFPACzBoh4NSL/BKFmbw5jefQ11n80bGgQL7ijaKp0+1CdDo+0/My6RnpbbGK +lcKpPLTtkSL7/xD3YMbFZ/XbXBu14j8G5zZ3rD+QMYle+tVOJz5TALSEhqWZraS4 +eNnjUaodChAzolZep6fIL3lyy/2rWs0QmWxLR92NjfSp8C2yw2c0X9FrsTXiCiWS +N+1+Pd3j+IRH0r+p9BPvTWRIXHotm6MmhOhyQarj4+6bE6JYcQST7bu4d9c2ng9c +VyWNWh0MgVMeBxIZ5EIbX8oZSOHhAQco0lqazydNc5t2A4KzvSrkog4EH7Bm4Tqc +b0YiqL4A7Ars0qzVRngd+/Xmpzx+zJKTRw/klb3RzsfGmrzgcdMqLJh9vcwV74Et +m7M5+q8wwyQkj+u1c8YS9bqCbo7R3aw8iImg9AGiP0Tx646AspUUm9FO7SnASxb1 +e5+Hen0ggrMlX3iopUBHApmhkPYODKTlh0JyyVFidnDAJ5QV4RhAiTGdRE7GQMuv +2E9NnOk6Tkag5QAMiI9i8IhfMF3805OJAmvoindRT2cLYy1EjK42ohhkIOuD1SK/ +15NbNFe32f2QuHlkpk8xpj2yzJwk/tBBGAUqmmB1MjP1xnVrLlGPp/Dowq7lU6zk +iH+jzIL2EAIMjfHLarJAhVzzmY+hpotUTku5iBzYDIwjL8k9nB44WnYhkTJhqp6G +IF85G31SXo5qUXguZ31/yZoaepmi26uZH+737V4ni39JuI/5KXqZCrlkfO0IkSZH +X0Lmzgg5m6gVvilCdN67CYz4Px433/tj89FNyKqBodo2v7WekkHCvZEo52skwgCm +t5C/G4HZeGui8DErK3e+ePZd5aQ6KdVMMBZF46MpAgakLRtgHO63AjI9U8SjohwP +7AnVRaK7dTnueMW/00FdtK1QGBdeLcuiWdEKUs4NBrl00SuAXgadaeGkRuOtqOBx +0aKfKtRlFHQ2shUR8eixKwtGx1awQ25xo4HMfI8xk+waN4ieWiMNjNaXGUHLYF7f +qIxURNS/RSzpQevoDHg/lYzgiVtvqgEmH8mjURHq91MU9iM0qn7i05Yn33zFl5y9 +AHavhhM8qFDJ14LefTEAx39aJ0ZdeskBVPzYpXv6qlA4uDscJkOUuDG2vJxIlSnb +GeM1yqmbCrtYqJv5ygfNTQ+xycnwZAcRcxkdjenJ1XJscj8T2jUJAfL7qEUp+fMO +AodfQLZL40THoCy6AFZlFSy2mvr1yZ995Z7dyq30HpJE7BqH/z/4HpMn1rjTT/aE +-----END RSA PRIVATE KEY----- From 0154052335b19dcede19f5ac274252c7073e0f77 Mon Sep 17 00:00:00 2001 From: Josh Soref Date: Thu, 31 Jan 2019 07:09:36 -0500 Subject: [PATCH 094/100] spelling: java script -- not JavaScript (#37057) --- .../org/elasticsearch/gradle/BuildPlugin.groovy | 12 ++++++------ x-pack/plugin/sql/qa/security/with-ssl/build.gradle | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 893f767e26e3f..f240ebb52c8ba 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -150,7 +150,7 @@ class BuildPlugin implements Plugin { } String inFipsJvmScript = 'print(java.security.Security.getProviders()[0].name.toLowerCase().contains("fips"));' - boolean inFipsJvm = Boolean.parseBoolean(runJavascript(project, runtimeJavaHome, inFipsJvmScript)) + boolean inFipsJvm = Boolean.parseBoolean(runJavaAsScript(project, runtimeJavaHome, inFipsJvmScript)) // Build debugging info println '=======================================' @@ -438,28 +438,28 @@ class BuildPlugin implements Plugin { String versionInfoScript = 'print(' + 'java.lang.System.getProperty("java.vendor") + " " + java.lang.System.getProperty("java.version") + ' + '" [" + java.lang.System.getProperty("java.vm.name") + " " + java.lang.System.getProperty("java.vm.version") + "]");' - return runJavascript(project, javaHome, versionInfoScript).trim() + return runJavaAsScript(project, javaHome, versionInfoScript).trim() } /** Finds the parsable java specification version */ private static String findJavaSpecificationVersion(Project project, String javaHome) { String versionScript = 'print(java.lang.System.getProperty("java.specification.version"));' - return runJavascript(project, javaHome, versionScript) + return runJavaAsScript(project, javaHome, versionScript) } private static String findJavaVendor(Project project, String javaHome) { String vendorScript = 'print(java.lang.System.getProperty("java.vendor"));' - return runJavascript(project, javaHome, vendorScript) + return runJavaAsScript(project, javaHome, vendorScript) } /** Finds the parsable java specification version */ private static String findJavaVersion(Project project, String javaHome) { String versionScript = 'print(java.lang.System.getProperty("java.version"));' - return runJavascript(project, javaHome, versionScript) + return runJavaAsScript(project, javaHome, versionScript) } /** Runs the given javascript using jjs from the jdk, and returns the output */ - private static String runJavascript(Project project, String javaHome, String script) { + private static String runJavaAsScript(Project project, String javaHome, String script) { ByteArrayOutputStream stdout = new ByteArrayOutputStream() ByteArrayOutputStream stderr = new ByteArrayOutputStream() if (Os.isFamily(Os.FAMILY_WINDOWS)) { diff --git a/x-pack/plugin/sql/qa/security/with-ssl/build.gradle b/x-pack/plugin/sql/qa/security/with-ssl/build.gradle index 483ba513b5fa1..de4e173463612 100644 --- a/x-pack/plugin/sql/qa/security/with-ssl/build.gradle +++ b/x-pack/plugin/sql/qa/security/with-ssl/build.gradle @@ -208,7 +208,7 @@ integTestCluster { } } Closure notRunningFips = { - Boolean.parseBoolean(BuildPlugin.runJavascript(project, project.runtimeJavaHome, + Boolean.parseBoolean(BuildPlugin.runJavaAsScript(project, project.runtimeJavaHome, 'print(java.security.Security.getProviders()[0].name.toLowerCase().contains("fips"));')) == false } From 3c439d3b92afa1b32b8123ebefbe0755a6280f4e Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 31 Jan 2019 13:52:09 +0100 Subject: [PATCH 095/100] Fix test bug when testing the merging of mappings and templates. (#38021) This test performs a typed index call when it actually means to run a typeless index call. --- .../test/indices.create/20_mix_typeless_typeful.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/20_mix_typeless_typeful.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/20_mix_typeless_typeful.yml index d196d5e5dd8c2..bb9157fe684f8 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/20_mix_typeless_typeful.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/20_mix_typeless_typeful.yml @@ -123,7 +123,6 @@ - do: index: index: test-1 - type: my_type body: { bar: 42 } - do: From a536fa7755dcb3716308f2b6e35662013db3a21e Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 31 Jan 2019 13:57:42 +0100 Subject: [PATCH 096/100] Treat put-mapping calls with `_doc` as a top-level key as typed calls. (#38032) Currently the put-mapping API assumes that because the type name is `_doc` then it is dealing with a typeless put-mapping call. Yet we still allow running the put-mapping API in a typed fashion with `_doc` as a type name. The current logic triggers surprising errors when doing a typed put-mapping call with `_doc` as a type name on an index that has a type already. This is a bit of a corner-case, but is more important on 6.x due to the fact that using the index API with `_doc` as a type name triggers typed calls to the put-mapping API with `_doc` as a type name. --- .../20_mix_typeless_typeful.yml | 35 ++++++++++++++++++- .../metadata/MetaDataMappingService.java | 25 ++++++++++--- 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/20_mix_typeless_typeful.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/20_mix_typeless_typeful.yml index 3aedff101110b..d964a382137f8 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/20_mix_typeless_typeful.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/20_mix_typeless_typeful.yml @@ -39,12 +39,45 @@ type: "keyword" # also test no-op updates that trigger special logic wrt the mapping version - do: - catch: bad_request + catch: /the final mapping would have more than 1 type/ indices.put_mapping: include_type_name: true index: index + type: some_other_type body: some_other_type: properties: bar: type: "long" + + +--- +"PUT mapping with _doc on an index that has types": + + - skip: + version: " - 6.99.99" + reason: Backport first + + + - do: + indices.create: + include_type_name: true + index: index + body: + mappings: + my_type: + properties: + foo: + type: "keyword" + + - do: + catch: /the final mapping would have more than 1 type/ + indices.put_mapping: + include_type_name: true + index: index + type: _doc + body: + _doc: + properties: + bar: + type: "long" diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java index 06dda07d2289e..cf31401983a62 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java @@ -37,6 +37,8 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.DocumentMapper; @@ -277,7 +279,8 @@ private ClusterState applyRequest(ClusterState currentState, PutMappingClusterSt if (mappingType == null) { mappingType = newMapper.type(); } else if (mappingType.equals(newMapper.type()) == false - && mapperService.resolveDocumentType(mappingType).equals(newMapper.type()) == false) { + && (isMappingSourceTyped(mapperService, mappingUpdateSource, request.type()) + || mapperService.resolveDocumentType(mappingType).equals(newMapper.type()) == false)) { throw new InvalidTypeNameException("Type name provided does not match type name within mapping definition."); } } @@ -297,10 +300,13 @@ private ClusterState applyRequest(ClusterState currentState, PutMappingClusterSt final Index index = indexMetaData.getIndex(); final MapperService mapperService = indexMapperServices.get(index); - // If the user gave _doc as a special type value or if they are using the new typeless APIs, - // then we apply the mapping update to the existing type. This allows to move to typeless - // APIs with indices whose type name is different from `_doc`. - String typeForUpdate = mapperService.resolveDocumentType(mappingType); // the type to use to apply the mapping update + // If the _type name is _doc and there is no _doc top-level key then this means that we + // are handling a typeless call. In such a case, we override _doc with the actual type + // name in the mappings. This allows to use typeless APIs on typed indices. + String typeForUpdate = mappingType; // the type to use to apply the mapping update + if (isMappingSourceTyped(mapperService, mappingUpdateSource, request.type()) == false) { + typeForUpdate = mapperService.resolveDocumentType(mappingType); + } CompressedXContent existingSource = null; DocumentMapper existingMapper = mapperService.documentMapper(typeForUpdate); @@ -365,6 +371,15 @@ public String describeTasks(List tasks) { } } + /** + * Returns {@code true} if the given {@code mappingSource} includes a type + * as a top-level object. + */ + private static boolean isMappingSourceTyped(MapperService mapperService, CompressedXContent mappingSource, String type) { + Map root = XContentHelper.convertToMap(mappingSource.compressedReference(), true, XContentType.JSON).v2(); + return root.size() == 1 && root.keySet().iterator().next().equals(type); + } + public void putMapping(final PutMappingClusterStateUpdateRequest request, final ActionListener listener) { clusterService.submitStateUpdateTask("put-mapping", request, From 9f026bb8ad266a73da015a06fcf45d43417f0537 Mon Sep 17 00:00:00 2001 From: Alexander Reelsen Date: Thu, 31 Jan 2019 14:18:28 +0100 Subject: [PATCH 097/100] Reduce object creation in Rounding class (#38061) This reduces objects creations in the rounding class (used by aggs) by properly creating the objects only once. Furthermore a few unneeded ZonedDateTime objects were created in order to create other objects out of them. This was changed as well. Running the benchmarks shows a much faster performance for all of the java time based Rounding classes. --- .../benchmark/time/RoundingBenchmark.java | 97 ++++++++++++++ .../org/elasticsearch/common/Rounding.java | 126 ++++++++---------- 2 files changed, 150 insertions(+), 73 deletions(-) create mode 100644 benchmarks/src/main/java/org/elasticsearch/benchmark/time/RoundingBenchmark.java diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/time/RoundingBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/time/RoundingBenchmark.java new file mode 100644 index 0000000000000..6da6d5290bfee --- /dev/null +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/time/RoundingBenchmark.java @@ -0,0 +1,97 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.benchmark.time; + +import org.elasticsearch.common.Rounding; +import org.elasticsearch.common.rounding.DateTimeUnit; +import org.elasticsearch.common.time.DateUtils; +import org.elasticsearch.common.unit.TimeValue; +import org.joda.time.DateTimeZone; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import java.time.ZoneId; +import java.util.concurrent.TimeUnit; + +@Fork(3) +@Warmup(iterations = 10) +@Measurement(iterations = 10) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +@SuppressWarnings("unused") //invoked by benchmarking framework +public class RoundingBenchmark { + + private final ZoneId zoneId = ZoneId.of("Europe/Amsterdam"); + private final DateTimeZone timeZone = DateUtils.zoneIdToDateTimeZone(zoneId); + + private final org.elasticsearch.common.rounding.Rounding jodaRounding = + org.elasticsearch.common.rounding.Rounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(timeZone).build(); + private final Rounding javaRounding = Rounding.builder(Rounding.DateTimeUnit.HOUR_OF_DAY) + .timeZone(zoneId).build(); + + private final org.elasticsearch.common.rounding.Rounding jodaDayOfMonthRounding = + org.elasticsearch.common.rounding.Rounding.builder(DateTimeUnit.DAY_OF_MONTH).timeZone(timeZone).build(); + private final Rounding javaDayOfMonthRounding = Rounding.builder(TimeValue.timeValueMinutes(60)) + .timeZone(zoneId).build(); + + private final org.elasticsearch.common.rounding.Rounding timeIntervalRoundingJoda = + org.elasticsearch.common.rounding.Rounding.builder(DateTimeUnit.DAY_OF_MONTH).timeZone(timeZone).build(); + private final Rounding timeIntervalRoundingJava = Rounding.builder(TimeValue.timeValueMinutes(60)) + .timeZone(zoneId).build(); + + private final long timestamp = 1548879021354L; + + @Benchmark + public long timeRoundingDateTimeUnitJoda() { + return jodaRounding.round(timestamp); + } + + @Benchmark + public long timeRoundingDateTimeUnitJava() { + return javaRounding.round(timestamp); + } + + @Benchmark + public long timeRoundingDateTimeUnitDayOfMonthJoda() { + return jodaDayOfMonthRounding.round(timestamp); + } + + @Benchmark + public long timeRoundingDateTimeUnitDayOfMonthJava() { + return javaDayOfMonthRounding.round(timestamp); + } + + @Benchmark + public long timeIntervalRoundingJava() { + return timeIntervalRoundingJava.round(timestamp); + } + + @Benchmark + public long timeIntervalRoundingJoda() { + return timeIntervalRoundingJoda.round(timestamp); + } +} diff --git a/server/src/main/java/org/elasticsearch/common/Rounding.java b/server/src/main/java/org/elasticsearch/common/Rounding.java index dab29c88634e9..6d11739133dda 100644 --- a/server/src/main/java/org/elasticsearch/common/Rounding.java +++ b/server/src/main/java/org/elasticsearch/common/Rounding.java @@ -27,8 +27,8 @@ import org.elasticsearch.common.unit.TimeValue; import java.io.IOException; -import java.time.DayOfWeek; import java.time.Instant; +import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; import java.time.OffsetDateTime; @@ -39,7 +39,9 @@ import java.time.temporal.ChronoUnit; import java.time.temporal.IsoFields; import java.time.temporal.TemporalField; +import java.time.temporal.TemporalQueries; import java.time.zone.ZoneOffsetTransition; +import java.time.zone.ZoneRules; import java.util.List; import java.util.Objects; @@ -185,13 +187,11 @@ static class TimeUnitRounding extends Rounding { TimeUnitRounding(DateTimeUnit unit, ZoneId timeZone) { this.unit = unit; this.timeZone = timeZone; - this.unitRoundsToMidnight = this.unit.field.getBaseUnit().getDuration().toMillis() > 60L * 60L * 1000L; + this.unitRoundsToMidnight = this.unit.field.getBaseUnit().getDuration().toMillis() > 3600000L; } TimeUnitRounding(StreamInput in) throws IOException { - unit = DateTimeUnit.resolve(in.readByte()); - timeZone = DateUtils.of(in.readString()); - unitRoundsToMidnight = unit.getField().getBaseUnit().getDuration().toMillis() > 60L * 60L * 1000L; + this(DateTimeUnit.resolve(in.readByte()), DateUtils.of(in.readString())); } @Override @@ -200,85 +200,67 @@ public byte id() { } private LocalDateTime truncateLocalDateTime(LocalDateTime localDateTime) { - localDateTime = localDateTime.withNano(0); - assert localDateTime.getNano() == 0; - if (unit.equals(DateTimeUnit.SECOND_OF_MINUTE)) { - return localDateTime; - } + switch (unit) { + case SECOND_OF_MINUTE: + return localDateTime.withNano(0); - localDateTime = localDateTime.withSecond(0); - assert localDateTime.getSecond() == 0; - if (unit.equals(DateTimeUnit.MINUTES_OF_HOUR)) { - return localDateTime; - } + case MINUTES_OF_HOUR: + return LocalDateTime.of(localDateTime.getYear(), localDateTime.getMonthValue(), localDateTime.getDayOfMonth(), + localDateTime.getHour(), localDateTime.getMinute(), 0, 0); - localDateTime = localDateTime.withMinute(0); - assert localDateTime.getMinute() == 0; - if (unit.equals(DateTimeUnit.HOUR_OF_DAY)) { - return localDateTime; - } + case HOUR_OF_DAY: + return LocalDateTime.of(localDateTime.getYear(), localDateTime.getMonth(), localDateTime.getDayOfMonth(), + localDateTime.getHour(), 0, 0); - localDateTime = localDateTime.withHour(0); - assert localDateTime.getHour() == 0; - if (unit.equals(DateTimeUnit.DAY_OF_MONTH)) { - return localDateTime; - } + case DAY_OF_MONTH: + LocalDate localDate = localDateTime.query(TemporalQueries.localDate()); + return localDate.atStartOfDay(); - if (unit.equals(DateTimeUnit.WEEK_OF_WEEKYEAR)) { - localDateTime = localDateTime.with(ChronoField.DAY_OF_WEEK, 1); - assert localDateTime.getDayOfWeek() == DayOfWeek.MONDAY; - return localDateTime; - } + case WEEK_OF_WEEKYEAR: + return LocalDateTime.of(localDateTime.toLocalDate(), LocalTime.MIDNIGHT).with(ChronoField.DAY_OF_WEEK, 1); - localDateTime = localDateTime.withDayOfMonth(1); - assert localDateTime.getDayOfMonth() == 1; - if (unit.equals(DateTimeUnit.MONTH_OF_YEAR)) { - return localDateTime; - } + case MONTH_OF_YEAR: + return LocalDateTime.of(localDateTime.getYear(), localDateTime.getMonthValue(), 1, 0, 0); - if (unit.equals(DateTimeUnit.QUARTER_OF_YEAR)) { - int quarter = (int) IsoFields.QUARTER_OF_YEAR.getFrom(localDateTime); - int month = ((quarter - 1) * 3) + 1; - localDateTime = localDateTime.withMonth(month); - assert localDateTime.getMonthValue() % 3 == 1; - return localDateTime; - } + case QUARTER_OF_YEAR: + int quarter = (int) IsoFields.QUARTER_OF_YEAR.getFrom(localDateTime); + int month = ((quarter - 1) * 3) + 1; + return LocalDateTime.of(localDateTime.getYear(), month, 1, 0, 0); - if (unit.equals(DateTimeUnit.YEAR_OF_CENTURY)) { - localDateTime = localDateTime.withMonth(1); - assert localDateTime.getMonthValue() == 1; - return localDateTime; - } + case YEAR_OF_CENTURY: + return LocalDateTime.of(LocalDate.of(localDateTime.getYear(), 1, 1), LocalTime.MIDNIGHT); - throw new IllegalArgumentException("NOT YET IMPLEMENTED for unit " + unit); + default: + throw new IllegalArgumentException("NOT YET IMPLEMENTED for unit " + unit); + } } @Override - public long round(long utcMillis) { + public long round(final long utcMillis) { + Instant instant = Instant.ofEpochMilli(utcMillis); if (unitRoundsToMidnight) { - final ZonedDateTime zonedDateTime = Instant.ofEpochMilli(utcMillis).atZone(timeZone); - final LocalDateTime localDateTime = zonedDateTime.toLocalDateTime(); + final LocalDateTime localDateTime = LocalDateTime.ofInstant(instant, timeZone); final LocalDateTime localMidnight = truncateLocalDateTime(localDateTime); return firstTimeOnDay(localMidnight); } else { + final ZoneRules rules = timeZone.getRules(); while (true) { - final Instant truncatedTime = truncateAsLocalTime(utcMillis); - final ZoneOffsetTransition previousTransition = timeZone.getRules().previousTransition(Instant.ofEpochMilli(utcMillis)); + final Instant truncatedTime = truncateAsLocalTime(instant, rules); + final ZoneOffsetTransition previousTransition = rules.previousTransition(instant); if (previousTransition == null) { // truncateAsLocalTime cannot have failed if there were no previous transitions return truncatedTime.toEpochMilli(); } - final long previousTransitionMillis = previousTransition.getInstant().toEpochMilli(); - - if (truncatedTime != null && previousTransitionMillis <= truncatedTime.toEpochMilli()) { + Instant previousTransitionInstant = previousTransition.getInstant(); + if (truncatedTime != null && previousTransitionInstant.compareTo(truncatedTime) < 1) { return truncatedTime.toEpochMilli(); } // There was a transition in between the input time and the truncated time. Return to the transition time and // round that down instead. - utcMillis = previousTransitionMillis - 1; + instant = previousTransitionInstant.minusNanos(1_000_000); } } } @@ -289,7 +271,7 @@ private long firstTimeOnDay(LocalDateTime localMidnight) { // Now work out what localMidnight actually means final List currentOffsets = timeZone.getRules().getValidOffsets(localMidnight); - if (currentOffsets.size() >= 1) { + if (currentOffsets.isEmpty() == false) { // There is at least one midnight on this day, so choose the first final ZoneOffset firstOffset = currentOffsets.get(0); final OffsetDateTime offsetMidnight = localMidnight.atOffset(firstOffset); @@ -302,23 +284,23 @@ private long firstTimeOnDay(LocalDateTime localMidnight) { } } - private Instant truncateAsLocalTime(long utcMillis) { + private Instant truncateAsLocalTime(Instant instant, final ZoneRules rules) { assert unitRoundsToMidnight == false : "truncateAsLocalTime should not be called if unitRoundsToMidnight"; - final LocalDateTime truncatedLocalDateTime - = truncateLocalDateTime(Instant.ofEpochMilli(utcMillis).atZone(timeZone).toLocalDateTime()); - final List currentOffsets = timeZone.getRules().getValidOffsets(truncatedLocalDateTime); + LocalDateTime localDateTime = LocalDateTime.ofInstant(instant, timeZone); + final LocalDateTime truncatedLocalDateTime = truncateLocalDateTime(localDateTime); + final List currentOffsets = rules.getValidOffsets(truncatedLocalDateTime); - if (currentOffsets.size() >= 1) { + if (currentOffsets.isEmpty() == false) { // at least one possibilities - choose the latest one that's still no later than the input time for (int offsetIndex = currentOffsets.size() - 1; offsetIndex >= 0; offsetIndex--) { final Instant result = truncatedLocalDateTime.atOffset(currentOffsets.get(offsetIndex)).toInstant(); - if (result.toEpochMilli() <= utcMillis) { + if (result.isAfter(instant) == false) { return result; } } - assert false : "rounded time not found for " + utcMillis + " with " + this; + assert false : "rounded time not found for " + instant + " with " + this; return null; } else { // The chosen local time didn't happen. This means we were given a time in an hour (or a minute) whose start @@ -328,7 +310,7 @@ private Instant truncateAsLocalTime(long utcMillis) { } private LocalDateTime nextRelevantMidnight(LocalDateTime localMidnight) { - assert localMidnight.toLocalTime().equals(LocalTime.of(0, 0, 0)) : "nextRelevantMidnight should only be called at midnight"; + assert localMidnight.toLocalTime().equals(LocalTime.MIDNIGHT) : "nextRelevantMidnight should only be called at midnight"; assert unitRoundsToMidnight : "firstTimeOnDay should only be called if unitRoundsToMidnight"; switch (unit) { @@ -350,8 +332,7 @@ private LocalDateTime nextRelevantMidnight(LocalDateTime localMidnight) { @Override public long nextRoundingValue(long utcMillis) { if (unitRoundsToMidnight) { - final ZonedDateTime zonedDateTime = Instant.ofEpochMilli(utcMillis).atZone(timeZone); - final LocalDateTime localDateTime = zonedDateTime.toLocalDateTime(); + final LocalDateTime localDateTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(utcMillis), timeZone); final LocalDateTime earlierLocalMidnight = truncateLocalDateTime(localDateTime); final LocalDateTime localMidnight = nextRelevantMidnight(earlierLocalMidnight); return firstTimeOnDay(localMidnight); @@ -433,14 +414,14 @@ public byte id() { @Override public long round(final long utcMillis) { final Instant utcInstant = Instant.ofEpochMilli(utcMillis); - final LocalDateTime rawLocalDateTime = Instant.ofEpochMilli(utcMillis).atZone(timeZone).toLocalDateTime(); + final LocalDateTime rawLocalDateTime = LocalDateTime.ofInstant(utcInstant, timeZone); // a millisecond value with the same local time, in UTC, as `utcMillis` has in `timeZone` final long localMillis = utcMillis + timeZone.getRules().getOffset(utcInstant).getTotalSeconds() * 1000; assert localMillis == rawLocalDateTime.toInstant(ZoneOffset.UTC).toEpochMilli(); final long roundedMillis = roundKey(localMillis, interval) * interval; - final LocalDateTime roundedLocalDateTime = Instant.ofEpochMilli(roundedMillis).atZone(ZoneOffset.UTC).toLocalDateTime(); + final LocalDateTime roundedLocalDateTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(roundedMillis), ZoneOffset.UTC); // Now work out what roundedLocalDateTime actually means final List currentOffsets = timeZone.getRules().getValidOffsets(roundedLocalDateTime); @@ -485,9 +466,8 @@ private static long roundKey(long value, long interval) { @Override public long nextRoundingValue(long time) { int offsetSeconds = timeZone.getRules().getOffset(Instant.ofEpochMilli(time)).getTotalSeconds(); - return ZonedDateTime.ofInstant(Instant.ofEpochMilli(time), ZoneOffset.UTC) - .plusSeconds(offsetSeconds) - .plusNanos(interval * 1_000_000) + long millis = time + interval + offsetSeconds * 1000; + return ZonedDateTime.ofInstant(Instant.ofEpochMilli(millis), ZoneOffset.UTC) .withZoneSameLocal(timeZone) .toInstant().toEpochMilli(); } From ae9f4df361fd4e8848c47593f0e359db860c76c6 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Thu, 31 Jan 2019 14:35:11 +0100 Subject: [PATCH 098/100] Don't Assert Ack on when Publish Timeout is 0 in Test (#38077) * Publish timeout is set to `0` so out of order processing of states on the node can lead to a `false` ack response * See #30672 * Closes #36813 --- .../elasticsearch/indices/state/RareClusterStateIT.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java b/server/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java index a34312b847e3b..d2f65d1168da8 100644 --- a/server/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java +++ b/server/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java @@ -333,12 +333,12 @@ public void testDelayedMappingPropagationOnReplica() throws Exception { // Force allocation of the primary on the master node by first only allocating on the master // and then allowing all nodes so that the replica gets allocated on the other node - assertAcked(prepareCreate("index").setSettings(Settings.builder() + prepareCreate("index").setSettings(Settings.builder() .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) - .put("index.routing.allocation.include._name", master)).get()); - assertAcked(client().admin().indices().prepareUpdateSettings("index").setSettings(Settings.builder() - .put("index.routing.allocation.include._name", "")).get()); + .put("index.routing.allocation.include._name", master)).get(); + client().admin().indices().prepareUpdateSettings("index").setSettings(Settings.builder() + .put("index.routing.allocation.include._name", "")).get(); ensureGreen(); // Check routing tables From 622fb7883b4e7c6de6dce2636108e334cca28cd3 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Thu, 31 Jan 2019 15:12:14 +0100 Subject: [PATCH 099/100] Introduce ability to minimize round-trips in CCS (#37828) With #37566 we have introduced the ability to merge multiple search responses into one. That makes it possible to expose a new way of executing cross-cluster search requests, that makes CCS much faster whenever there is network latency between the CCS coordinating node and the remote clusters. The coordinating node can now send a single search request to each remote cluster, which gets reduced by each one of them. from + size results are requested to each cluster, and the reduce phase in each cluster is non final (meaning that buckets are not pruned and pipeline aggs are not executed). The CCS coordinating node performs an additional, final reduction, which produces one search response out of the multiple responses received from the different clusters. This new execution path will be activated by default for any CCS request unless a scroll is provided or inner hits are requested as part of field collapsing. The search API accepts now a new parameter called ccs_minimize_roundtrips that allows to opt-out of the default behaviour. Relates to #32125 --- .../client/RequestConverters.java | 1 + .../client/RequestConvertersTests.java | 6 +- .../modules/cross-cluster-search.asciidoc | 43 ++ docs/reference/search/request-body.asciidoc | 5 + .../mustache/MultiSearchTemplateRequest.java | 1 - .../MultiSearchTemplateResponseTests.java | 18 +- ...rossClusterSearchUnavailableClusterIT.java | 13 + .../test/multi_cluster/10_basic.yml | 26 + .../test/multi_cluster/70_skip_shards.yml | 4 + .../resources/rest-api-spec/api/msearch.json | 5 + .../rest-api-spec/api/msearch_template.json | 5 + .../resources/rest-api-spec/api/search.json | 5 + .../rest-api-spec/api/search_template.json | 5 + .../action/search/MultiSearchRequest.java | 7 + .../action/search/SearchRequest.java | 138 ++-- .../action/search/SearchResponse.java | 3 +- .../action/search/SearchResponseMerger.java | 107 +++- .../action/search/TransportSearchAction.java | 253 ++++++-- .../common/io/stream/StreamInput.java | 2 +- .../common/io/stream/StreamOutput.java | 2 +- .../action/search/RestMultiSearchAction.java | 3 +- .../rest/action/search/RestSearchAction.java | 1 + .../elasticsearch/search/SearchService.java | 6 +- .../search/MultiSearchRequestTests.java | 12 +- .../search/MultiSearchResponseTests.java | 7 +- .../action/search/SearchRequestTests.java | 31 +- .../search/SearchResponseMergerTests.java | 61 +- .../action/search/SearchResponseTests.java | 37 +- .../TransportSearchActionSingleNodeTests.java | 11 +- .../search/TransportSearchActionTests.java | 591 ++++++++++++++---- .../common/io/stream/BytesStreamsTests.java | 2 +- .../transport/RemoteClusterClientTests.java | 6 +- .../RemoteClusterConnectionTests.java | 27 + .../search/RandomSearchRequestGenerator.java | 15 +- .../test/multi_cluster/10_basic.yml | 22 + .../test/multi_cluster/40_scroll.yml | 7 + .../test/multi_cluster/60_skip_shards.yml | 2 + 37 files changed, 1159 insertions(+), 331 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index 1286e083d8203..a30fec41b0bf3 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -399,6 +399,7 @@ private static void addSearchRequestParams(Params params, SearchRequest searchRe params.withPreference(searchRequest.preference()); params.withIndicesOptions(searchRequest.indicesOptions()); params.putParam("search_type", searchRequest.searchType().name().toLowerCase(Locale.ROOT)); + params.putParam("ccs_minimize_roundtrips", Boolean.toString(searchRequest.isCcsMinimizeRoundtrips())); if (searchRequest.requestCache() != null) { params.putParam("request_cache", Boolean.toString(searchRequest.requestCache())); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index f31c562332687..95971ad40ced0 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -1239,7 +1239,7 @@ public void testMultiSearch() throws IOException { requests.add(searchRequest); }; MultiSearchRequest.readMultiLineFormat(new BytesArray(EntityUtils.toByteArray(request.getEntity())), - REQUEST_BODY_CONTENT_TYPE.xContent(), consumer, null, multiSearchRequest.indicesOptions(), null, null, null, + REQUEST_BODY_CONTENT_TYPE.xContent(), consumer, null, multiSearchRequest.indicesOptions(), null, null, null, null, xContentRegistry(), true); assertEquals(requests, multiSearchRequest.requests()); } @@ -1862,6 +1862,10 @@ private static void setRandomSearchParams(SearchRequest searchRequest, searchRequest.scroll(randomTimeValue()); expectedParams.put("scroll", searchRequest.scroll().keepAlive().getStringRep()); } + if (randomBoolean()) { + searchRequest.setCcsMinimizeRoundtrips(randomBoolean()); + } + expectedParams.put("ccs_minimize_roundtrips", Boolean.toString(searchRequest.isCcsMinimizeRoundtrips())); } static void setRandomIndicesOptions(Consumer setter, Supplier getter, diff --git a/docs/reference/modules/cross-cluster-search.asciidoc b/docs/reference/modules/cross-cluster-search.asciidoc index 61b0bb50aedc7..b59f74198c3e8 100644 --- a/docs/reference/modules/cross-cluster-search.asciidoc +++ b/docs/reference/modules/cross-cluster-search.asciidoc @@ -65,6 +65,7 @@ GET /cluster_one:twitter/_search { "took": 150, "timed_out": false, + "num_reduce_phases": 2, "_shards": { "total": 1, "successful": 1, @@ -130,6 +131,7 @@ will be prefixed with their remote cluster name: { "took": 150, "timed_out": false, + "num_reduce_phases": 3, "_shards": { "total": 2, "successful": 2, @@ -222,6 +224,7 @@ GET /cluster_one:twitter,cluster_two:twitter,twitter/_search <1> { "took": 150, "timed_out": false, + "num_reduce_phases": 3, "_shards": { "total": 2, "successful": 2, @@ -273,3 +276,43 @@ GET /cluster_one:twitter,cluster_two:twitter,twitter/_search <1> // TESTRESPONSE[s/"_score": 1/"_score": "$body.hits.hits.0._score"/] // TESTRESPONSE[s/"_score": 2/"_score": "$body.hits.hits.1._score"/] <1> The `clusters` section indicates that one cluster was unavailable and got skipped + +[float] +[[ccs-reduction]] +=== CCS reduction phase + +Cross-cluster search requests can be executed in two ways: + +- the CCS coordinating node minimizes network round-trips by sending one search +request to each cluster. Each cluster performs the search independently, +reducing and fetching results. Once the CCS node has received all the +responses, it performs another reduction and returns the relevant results back +to the user. This strategy is beneficial when there is network latency between +the CCS coordinating node and the remote clusters involved, which is typically +the case. A single request is sent to each remote cluster, at the cost of +retrieving `from` + `size` already fetched results. This is the default +strategy, used whenever possible. In case a scroll is provided, or inner hits +are requested as part of field collapsing, this strategy is not supported hence +network round-trips cannot be minimized and the following strategy is used +instead. + +- the CCS coordinating node sends a <> request to +each remote cluster, in order to collect information about their corresponding +remote indices involved in the search request and the shards where their data +is located. Once each cluster has responded to such request, the search +executes as if all shards were part of the same cluster. The coordinating node +sends one request to each shard involved, each shard executes the query and +returns its own results which are then reduced (and fetched, depending on the +<>) by the CCS coordinating node. +This strategy may be beneficial whenever there is very low network latency +between the CCS coordinating node and the remote clusters involved, as it +treats all shards the same, at the cost of sending many requests to each remote +cluster, which is problematic in presence of network latency. + +The <> supports the `ccs_minimize_roundtrips` +parameter, which defaults to `true` and can be set to `false` in case +minimizing network round-trips is not desirable. + +Note that all the communication between the nodes, regardless of which cluster +they belong to and the selected reduce mode, happens through the +<>. diff --git a/docs/reference/search/request-body.asciidoc b/docs/reference/search/request-body.asciidoc index dac7622aab8ed..120c4c6757599 100644 --- a/docs/reference/search/request-body.asciidoc +++ b/docs/reference/search/request-body.asciidoc @@ -113,6 +113,11 @@ And here is a sample response: reduce the memory overhead per search request if the potential number of shards in the request can be large. +`ccs_minimize_roundtrips`:: + + Defaults to `true`. Set to `false` to disable minimizing network round-trips + between the coordinating node and the remote clusters when executing + cross-cluster search requests. See <> for more. Out of the above, the `search_type`, `request_cache` and the `allow_partial_search_results` diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java index a685c3ba5ba7c..c80f99484a947 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java @@ -65,7 +65,6 @@ public MultiSearchTemplateRequest add(SearchTemplateRequest request) { return this; } - /** * Returns the amount of search requests specified in this multi search requests are allowed to be ran concurrently. */ diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateResponseTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateResponseTests.java index 2c67dd4709bc9..dadaf7cb05a09 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateResponseTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateResponseTests.java @@ -50,7 +50,7 @@ protected MultiSearchTemplateResponse createTestInstance() { int successfulShards = randomIntBetween(0, totalShards); int skippedShards = totalShards - successfulShards; InternalSearchResponse internalSearchResponse = InternalSearchResponse.empty(); - SearchResponse.Clusters clusters = new SearchResponse.Clusters(totalShards, successfulShards, skippedShards); + SearchResponse.Clusters clusters = randomClusters(); SearchTemplateResponse searchTemplateResponse = new SearchTemplateResponse(); SearchResponse searchResponse = new SearchResponse(internalSearchResponse, null, totalShards, successfulShards, skippedShards, tookInMillis, ShardSearchFailure.EMPTY_ARRAY, clusters); @@ -59,7 +59,13 @@ protected MultiSearchTemplateResponse createTestInstance() { } return new MultiSearchTemplateResponse(items, overallTookInMillis); } - + + private static SearchResponse.Clusters randomClusters() { + int totalClusters = randomIntBetween(0, 10); + int successfulClusters = randomIntBetween(0, totalClusters); + int skippedClusters = totalClusters - successfulClusters; + return new SearchResponse.Clusters(totalClusters, successfulClusters, skippedClusters); + } private static MultiSearchTemplateResponse createTestInstanceWithFailures() { int numItems = randomIntBetween(0, 128); @@ -67,14 +73,13 @@ private static MultiSearchTemplateResponse createTestInstanceWithFailures() { MultiSearchTemplateResponse.Item[] items = new MultiSearchTemplateResponse.Item[numItems]; for (int i = 0; i < numItems; i++) { if (randomBoolean()) { - // Creating a minimal response is OK, because SearchResponse self - // is tested elsewhere. + // Creating a minimal response is OK, because SearchResponse is tested elsewhere. long tookInMillis = randomNonNegativeLong(); int totalShards = randomIntBetween(1, Integer.MAX_VALUE); int successfulShards = randomIntBetween(0, totalShards); int skippedShards = totalShards - successfulShards; InternalSearchResponse internalSearchResponse = InternalSearchResponse.empty(); - SearchResponse.Clusters clusters = new SearchResponse.Clusters(totalShards, successfulShards, skippedShards); + SearchResponse.Clusters clusters = randomClusters(); SearchTemplateResponse searchTemplateResponse = new SearchTemplateResponse(); SearchResponse searchResponse = new SearchResponse(internalSearchResponse, null, totalShards, successfulShards, skippedShards, tookInMillis, ShardSearchFailure.EMPTY_ARRAY, clusters); @@ -133,6 +138,5 @@ public void testFromXContentWithFailures() throws IOException { AbstractXContentTestCase.testFromXContent(NUMBER_OF_TEST_RUNS, instanceSupplier, supportsUnknownFields, Strings.EMPTY_ARRAY, getRandomFieldsExcludeFilterWhenResultHasErrors(), this::createParser, this::doParseInstance, this::assertEqualInstances, assertToXContentEquivalence, ToXContent.EMPTY_PARAMS); - } - + } } diff --git a/qa/ccs-unavailable-clusters/src/test/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java b/qa/ccs-unavailable-clusters/src/test/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java index efac9ac220573..e280b1d2d1a05 100644 --- a/qa/ccs-unavailable-clusters/src/test/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java +++ b/qa/ccs-unavailable-clusters/src/test/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java @@ -22,6 +22,7 @@ import org.apache.http.HttpEntity; import org.apache.http.entity.ContentType; import org.apache.http.nio.entity.NStringEntity; +import org.apache.lucene.search.TotalHits; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsAction; @@ -32,9 +33,11 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequest; +import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; @@ -49,6 +52,8 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.internal.InternalSearchResponse; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.threadpool.TestThreadPool; @@ -107,6 +112,14 @@ private static MockTransportService startTransport( channel.sendResponse(new ClusterSearchShardsResponse(new ClusterSearchShardsGroup[0], knownNodes.toArray(new DiscoveryNode[0]), Collections.emptyMap())); }); + newService.registerRequestHandler(SearchAction.NAME, ThreadPool.Names.SAME, SearchRequest::new, + (request, channel, task) -> { + InternalSearchResponse response = new InternalSearchResponse(new SearchHits(new SearchHit[0], + new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN), InternalAggregations.EMPTY, null, null, false, null, 1); + SearchResponse searchResponse = new SearchResponse(response, null, 1, 1, 0, 100, ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY); + channel.sendResponse(searchResponse); + }); newService.registerRequestHandler(ClusterStateAction.NAME, ThreadPool.Names.SAME, ClusterStateRequest::new, (request, channel, task) -> { DiscoveryNodes.Builder builder = DiscoveryNodes.builder(); diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yml index 5acf84139bbf4..4499a60bfe24a 100644 --- a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yml +++ b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yml @@ -36,6 +36,10 @@ terms: field: f1.keyword + - match: {_clusters.total: 2} + - match: {_clusters.successful: 2} + - match: {_clusters.skipped: 0} + - match: {num} - match: { _shards.total: 5 } - match: { hits.total: 11 } - gte: { hits.hits.0._seq_no: 0 } @@ -59,6 +63,9 @@ terms: field: f1.keyword + - match: {_clusters.total: 2} + - match: {_clusters.successful: 2} + - match: {_clusters.skipped: 0} - match: { _shards.total: 5 } - match: { hits.total: 6} - match: { hits.hits.0._index: "my_remote_cluster:test_index"} @@ -76,6 +83,9 @@ terms: field: f1.keyword + - match: {_clusters.total: 1} + - match: {_clusters.successful: 1} + - match: {_clusters.skipped: 0} - match: { _shards.total: 3 } - match: { hits.total: 6} - match: { hits.hits.0._index: "my_remote_cluster:test_index"} @@ -93,6 +103,7 @@ terms: field: f1.keyword + - is_false: _clusters - match: { _shards.total: 2 } - match: { hits.total: 5} - match: { hits.hits.0._index: "test_index"} @@ -122,6 +133,9 @@ rest_total_hits_as_int: true index: test_remote_cluster:test_index + - match: {_clusters.total: 1} + - match: {_clusters.successful: 1} + - match: {_clusters.skipped: 0} - match: { _shards.total: 3 } - match: { hits.total: 6 } - match: { hits.hits.0._index: "test_remote_cluster:test_index" } @@ -148,6 +162,9 @@ rest_total_hits_as_int: true index: "*:test_index" + - match: {_clusters.total: 2} + - match: {_clusters.successful: 2} + - match: {_clusters.skipped: 0} - match: { _shards.total: 6 } - match: { hits.total: 12 } @@ -159,6 +176,9 @@ rest_total_hits_as_int: true index: my_remote_cluster:aliased_test_index + - match: {_clusters.total: 1} + - match: {_clusters.successful: 1} + - match: {_clusters.skipped: 0} - match: { _shards.total: 3 } - match: { hits.total: 2 } - match: { hits.hits.0._source.filter_field: 1 } @@ -172,6 +192,9 @@ rest_total_hits_as_int: true index: my_remote_cluster:aliased_test_index,my_remote_cluster:field_caps_index_1 + - match: {_clusters.total: 1} + - match: {_clusters.successful: 1} + - match: {_clusters.skipped: 0} - match: { _shards.total: 4 } - match: { hits.total: 2 } - match: { hits.hits.0._source.filter_field: 1 } @@ -185,6 +208,9 @@ rest_total_hits_as_int: true index: "my_remote_cluster:single_doc_index" + - match: {_clusters.total: 1} + - match: {_clusters.successful: 1} + - match: {_clusters.skipped: 0} - match: { _shards.total: 1 } - match: { hits.total: 1 } - match: { hits.hits.0._index: "my_remote_cluster:single_doc_index"} diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/70_skip_shards.yml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/70_skip_shards.yml index 81c09da54c085..d1a5a273e1d0f 100644 --- a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/70_skip_shards.yml +++ b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/70_skip_shards.yml @@ -29,10 +29,12 @@ rest_total_hits_as_int: true index: "skip_shards_index,my_remote_cluster:single_doc_index" pre_filter_shard_size: 1 + ccs_minimize_roundtrips: false body: { "size" : 10, "query" : { "range" : { "created_at" : { "gte" : "2016-02-01", "lt": "2018-02-01"} } } } - match: { hits.total: 1 } - match: { hits.hits.0._index: "skip_shards_index"} + - is_false: num_reduce_phases - match: { _shards.total: 2 } - match: { _shards.successful: 2 } - match: { _shards.skipped : 1} @@ -45,10 +47,12 @@ rest_total_hits_as_int: true index: "skip_shards_index,my_remote_cluster:single_doc_index" pre_filter_shard_size: 1 + ccs_minimize_roundtrips: false body: { "size" : 10, "query" : { "range" : { "created_at" : { "gte" : "2015-02-01", "lt": "2016-02-01"} } } } - match: { hits.total: 1 } - match: { hits.hits.0._index: "my_remote_cluster:single_doc_index"} + - is_false: num_reduce_phases - match: { _shards.total: 2 } - match: { _shards.successful: 2 } - match: { _shards.skipped : 1} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json index 13a6005c9a189..398dcbd29515d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json @@ -43,6 +43,11 @@ "type" : "boolean", "description" : "Indicates whether hits.total should be rendered as an integer or an object in the rest search response", "default" : false + }, + "ccs_minimize_roundtrips": { + "type" : "boolean", + "description" : "Indicates whether network round-trips should be minimized as part of cross-cluster search requests execution", + "default" : "true" } } }, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json index e49cc2083f929..e89f96e06960f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json @@ -33,6 +33,11 @@ "type" : "boolean", "description" : "Indicates whether hits.total should be rendered as an integer or an object in the rest search response", "default" : false + }, + "ccs_minimize_roundtrips": { + "type" : "boolean", + "description" : "Indicates whether network round-trips should be minimized as part of cross-cluster search requests execution", + "default" : "true" } } }, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json index 9ac02b1214a2f..f44c0f74b2c3d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json @@ -24,6 +24,11 @@ "type" : "boolean", "description" : "Specify whether wildcard and prefix queries should be analyzed (default: false)" }, + "ccs_minimize_roundtrips": { + "type" : "boolean", + "description" : "Indicates whether network round-trips should be minimized as part of cross-cluster search requests execution", + "default" : "true" + }, "default_operator": { "type" : "enum", "options" : ["AND","OR"], diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json index 528ee905e7ee8..24b7fa135b331 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json @@ -67,6 +67,11 @@ "type" : "boolean", "description" : "Indicates whether hits.total should be rendered as an integer or an object in the rest search response", "default" : false + }, + "ccs_minimize_roundtrips": { + "type" : "boolean", + "description" : "Indicates whether network round-trips should be minimized as part of cross-cluster search requests execution", + "default" : "true" } } }, diff --git a/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java index a574c0559804b..1dc2dc624b277 100644 --- a/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java @@ -173,6 +173,7 @@ public static void readMultiLineFormat(BytesReference data, String[] types, String routing, String searchType, + Boolean ccsMinimizeRoundtrips, NamedXContentRegistry registry, boolean allowExplicitIndex) throws IOException { int from = 0; @@ -205,6 +206,9 @@ public static void readMultiLineFormat(BytesReference data, if (searchType != null) { searchRequest.searchType(searchType); } + if (ccsMinimizeRoundtrips != null) { + searchRequest.setCcsMinimizeRoundtrips(ccsMinimizeRoundtrips); + } IndicesOptions defaultOptions = searchRequest.indicesOptions(); // now parse the action if (nextMarker - from > 0) { @@ -226,6 +230,8 @@ public static void readMultiLineFormat(BytesReference data, searchRequest.types(nodeStringArrayValue(value)); } else if ("search_type".equals(entry.getKey()) || "searchType".equals(entry.getKey())) { searchRequest.searchType(nodeStringValue(value, null)); + } else if ("ccs_minimize_roundtrips".equals(entry.getKey()) || "ccsMinimizeRoundtrips".equals(entry.getKey())) { + searchRequest.setCcsMinimizeRoundtrips(nodeBooleanValue(value)); } else if ("request_cache".equals(entry.getKey()) || "requestCache".equals(entry.getKey())) { searchRequest.requestCache(nodeBooleanValue(value, entry.getKey())); } else if ("preference".equals(entry.getKey())) { @@ -327,6 +333,7 @@ public static void writeSearchRequestParams(SearchRequest request, XContentBuild if (request.searchType() != null) { xContentBuilder.field("search_type", request.searchType().name().toLowerCase(Locale.ROOT)); } + xContentBuilder.field("ccs_minimize_roundtrips", request.isCcsMinimizeRoundtrips()); if (request.requestCache() != null) { xContentBuilder.field("request_cache", request.requestCache()); } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java index 020887068f015..55122b6806fd2 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -93,6 +93,8 @@ public final class SearchRequest extends ActionRequest implements IndicesRequest private String[] types = Strings.EMPTY_ARRAY; + private boolean ccsMinimizeRoundtrips = true; + public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandOpenAndForbidClosedIgnoreThrottled(); private IndicesOptions indicesOptions = DEFAULT_INDICES_OPTIONS; @@ -106,21 +108,7 @@ public SearchRequest() { * Constructs a new search request from the provided search request */ public SearchRequest(SearchRequest searchRequest) { - this.allowPartialSearchResults = searchRequest.allowPartialSearchResults; - this.batchedReduceSize = searchRequest.batchedReduceSize; - this.indices = searchRequest.indices; - this.indicesOptions = searchRequest.indicesOptions; - this.maxConcurrentShardRequests = searchRequest.maxConcurrentShardRequests; - this.preference = searchRequest.preference; - this.preFilterShardSize = searchRequest.preFilterShardSize; - this.requestCache = searchRequest.requestCache; - this.routing = searchRequest.routing; - this.scroll = searchRequest.scroll; - this.searchType = searchRequest.searchType; - this.source = searchRequest.source; - this.types = searchRequest.types; - this.localClusterAlias = searchRequest.localClusterAlias; - this.absoluteStartMillis = searchRequest.absoluteStartMillis; + this(searchRequest, searchRequest.indices, searchRequest.localClusterAlias, searchRequest.absoluteStartMillis); } /** @@ -144,16 +132,40 @@ public SearchRequest(String[] indices, SearchSourceBuilder source) { } /** - * Creates a new search request by providing the alias of the cluster where it will be executed, as well as the current time in - * milliseconds from the epoch time. Used when a {@link SearchRequest} is created and executed as part of a cross-cluster search - * request performing local reduction on each cluster. The coordinating CCS node provides the alias to prefix index names with in - * the returned search results, and the current time to be used on the remote clusters to ensure that the same value is used. + * Creates a new search request by providing the search request to copy all fields from, the indices to search against, + * the alias of the cluster where it will be executed, as well as the start time in milliseconds from the epoch time. + * Used when a {@link SearchRequest} is created and executed as part of a cross-cluster search request performing local reduction + * on each cluster. The coordinating CCS node provides the original search request, the indices to search against as well as the + * alias to prefix index names with in the returned search results, and the absolute start time to be used on the remote clusters + * to ensure that the same value is used. */ - SearchRequest(String localClusterAlias, long absoluteStartMillis) { - this.localClusterAlias = Objects.requireNonNull(localClusterAlias, "cluster alias must not be null"); + static SearchRequest withLocalReduction(SearchRequest originalSearchRequest, String[] indices, + String localClusterAlias, long absoluteStartMillis) { + Objects.requireNonNull(originalSearchRequest, "search request must not be null"); + validateIndices(indices); + Objects.requireNonNull(localClusterAlias, "cluster alias must not be null"); if (absoluteStartMillis < 0) { throw new IllegalArgumentException("absoluteStartMillis must not be negative but was [" + absoluteStartMillis + "]"); } + return new SearchRequest(originalSearchRequest, indices, localClusterAlias, absoluteStartMillis); + } + + private SearchRequest(SearchRequest searchRequest, String[] indices, String localClusterAlias, long absoluteStartMillis) { + this.allowPartialSearchResults = searchRequest.allowPartialSearchResults; + this.batchedReduceSize = searchRequest.batchedReduceSize; + this.ccsMinimizeRoundtrips = searchRequest.ccsMinimizeRoundtrips; + this.indices = indices; + this.indicesOptions = searchRequest.indicesOptions; + this.maxConcurrentShardRequests = searchRequest.maxConcurrentShardRequests; + this.preference = searchRequest.preference; + this.preFilterShardSize = searchRequest.preFilterShardSize; + this.requestCache = searchRequest.requestCache; + this.routing = searchRequest.routing; + this.scroll = searchRequest.scroll; + this.searchType = searchRequest.searchType; + this.source = searchRequest.source; + this.types = searchRequest.types; + this.localClusterAlias = localClusterAlias; this.absoluteStartMillis = absoluteStartMillis; } @@ -191,6 +203,9 @@ public SearchRequest(StreamInput in) throws IOException { localClusterAlias = null; absoluteStartMillis = DEFAULT_ABSOLUTE_START_MILLIS; } + if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + ccsMinimizeRoundtrips = in.readBoolean(); + } } @Override @@ -217,33 +232,37 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(absoluteStartMillis); } } + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + out.writeBoolean(ccsMinimizeRoundtrips); + } } @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; - final Scroll scroll = scroll(); - if (source != null - && source.trackTotalHitsUpTo() != null - && source.trackTotalHitsUpTo() != SearchContext.TRACK_TOTAL_HITS_ACCURATE - && scroll != null) { - validationException = - addValidationError("disabling [track_total_hits] is not allowed in a scroll context", validationException); - } - if (source != null && source.from() > 0 && scroll != null) { - validationException = - addValidationError("using [from] is not allowed in a scroll context", validationException); - } - if (requestCache != null && requestCache && scroll != null) { - validationException = - addValidationError("[request_cache] cannot be used in a scroll context", validationException); - } - if (source != null && source.size() == 0 && scroll != null) { - validationException = addValidationError("[size] cannot be [0] in a scroll context", validationException); - } - if (source != null && source.rescores() != null && source.rescores().isEmpty() == false && scroll != null) { - validationException = - addValidationError("using [rescore] is not allowed in a scroll context", validationException); + boolean scroll = scroll() != null; + if (scroll) { + if (source != null) { + if (source.trackTotalHitsUpTo() != null && source.trackTotalHitsUpTo() != SearchContext.TRACK_TOTAL_HITS_ACCURATE) { + validationException = + addValidationError("disabling [track_total_hits] is not allowed in a scroll context", validationException); + } + if (source.from() > 0) { + validationException = + addValidationError("using [from] is not allowed in a scroll context", validationException); + } + if (source.size() == 0) { + validationException = addValidationError("[size] cannot be [0] in a scroll context", validationException); + } + if (source.rescores() != null && source.rescores().isEmpty() == false) { + validationException = + addValidationError("using [rescore] is not allowed in a scroll context", validationException); + } + } + if (requestCache != null && requestCache) { + validationException = + addValidationError("[request_cache] cannot be used in a scroll context", validationException); + } } return validationException; } @@ -261,8 +280,8 @@ String getLocalClusterAlias() { /** * Returns the current time in milliseconds from the time epoch, to be used for the execution of this search request. Used to * ensure that the same value, determined by the coordinating node, is used on all nodes involved in the execution of the search - * request. When created through {@link #SearchRequest(String, long)}, this method returns the provided current time, otherwise - * it will return {@link System#currentTimeMillis()}. + * request. When created through {@link #withLocalReduction(SearchRequest, String[], String, long)}, this method returns the provided + * current time, otherwise it will return {@link System#currentTimeMillis()}. * */ long getOrCreateAbsoluteStartMillis() { @@ -274,12 +293,16 @@ long getOrCreateAbsoluteStartMillis() { */ @Override public SearchRequest indices(String... indices) { + validateIndices(indices); + this.indices = indices; + return this; + } + + private static void validateIndices(String... indices) { Objects.requireNonNull(indices, "indices must not be null"); for (String index : indices) { Objects.requireNonNull(index, "index must not be null"); } - this.indices = indices; - return this; } @Override @@ -292,6 +315,21 @@ public SearchRequest indicesOptions(IndicesOptions indicesOptions) { return this; } + /** + * Returns whether network round-trips should be minimized when executing cross-cluster search requests. + * Defaults to true. + */ + public boolean isCcsMinimizeRoundtrips() { + return ccsMinimizeRoundtrips; + } + + /** + * Sets whether network round-trips should be minimized when executing cross-cluster search requests. Defaults to true. + */ + public void setCcsMinimizeRoundtrips(boolean ccsMinimizeRoundtrips) { + this.ccsMinimizeRoundtrips = ccsMinimizeRoundtrips; + } + /** * The document types to execute the search against. Defaults to be executed against * all types. @@ -583,14 +621,15 @@ public boolean equals(Object o) { Objects.equals(indicesOptions, that.indicesOptions) && Objects.equals(allowPartialSearchResults, that.allowPartialSearchResults) && Objects.equals(localClusterAlias, that.localClusterAlias) && - absoluteStartMillis == that.absoluteStartMillis; + absoluteStartMillis == that.absoluteStartMillis && + ccsMinimizeRoundtrips == that.ccsMinimizeRoundtrips; } @Override public int hashCode() { return Objects.hash(searchType, Arrays.hashCode(indices), routing, preference, source, requestCache, scroll, Arrays.hashCode(types), indicesOptions, batchedReduceSize, maxConcurrentShardRequests, preFilterShardSize, - allowPartialSearchResults, localClusterAlias, absoluteStartMillis); + allowPartialSearchResults, localClusterAlias, absoluteStartMillis, ccsMinimizeRoundtrips); } @Override @@ -610,6 +649,7 @@ public String toString() { ", allowPartialSearchResults=" + allowPartialSearchResults + ", localClusterAlias=" + localClusterAlias + ", getOrCreateAbsoluteStartMillis=" + absoluteStartMillis + + ", ccsMinimizeRoundtrips=" + ccsMinimizeRoundtrips + ", source=" + source + '}'; } } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java index 0273d5e58219a..dd0d4de07d6f4 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java @@ -111,7 +111,6 @@ public Aggregations getAggregations() { return internalResponse.aggregations(); } - public Suggest getSuggest() { return internalResponse.suggest(); } @@ -349,7 +348,7 @@ static SearchResponse innerFromXContent(XContentParser parser) throws IOExceptio SearchResponseSections searchResponseSections = new SearchResponseSections(hits, aggs, suggest, timedOut, terminatedEarly, profile, numReducePhases); return new SearchResponse(searchResponseSections, scrollId, totalShards, successfulShards, skippedShards, tookInMillis, - failures.toArray(new ShardSearchFailure[failures.size()]), clusters); + failures.toArray(ShardSearchFailure.EMPTY_ARRAY), clusters); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponseMerger.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponseMerger.java index b146d42c0d2e6..567040246c50f 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchResponseMerger.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponseMerger.java @@ -39,6 +39,7 @@ import org.elasticsearch.search.profile.ProfileShardResult; import org.elasticsearch.search.profile.SearchProfileShardResults; import org.elasticsearch.search.suggest.Suggest; +import org.elasticsearch.transport.RemoteClusterAware; import java.util.ArrayList; import java.util.Arrays; @@ -76,9 +77,9 @@ //from the remote clusters in the fetch phase. This would be identical to the removed QueryAndFetch strategy except that only the remote //cluster response would have the fetch results. final class SearchResponseMerger { - private final int from; - private final int size; - private final int trackTotalHitsUpTo; + final int from; + final int size; + final int trackTotalHitsUpTo; private final SearchTimeProvider searchTimeProvider; private final Function reduceContextFunction; private final List searchResponses = new CopyOnWriteArrayList<>(); @@ -98,15 +99,28 @@ final class SearchResponseMerger { * That may change in the future as it's possible to introduce incremental merges as responses come in if necessary. */ void add(SearchResponse searchResponse) { + assert searchResponse.getScrollId() == null : "merging scroll results is not supported"; searchResponses.add(searchResponse); } + int numResponses() { + return searchResponses.size(); + } + /** * Returns the merged response. To be called once all responses have been added through {@link #add(SearchResponse)} * so that all responses are merged into a single one. */ SearchResponse getMergedResponse(Clusters clusters) { - assert searchResponses.size() > 1; + //if the search is only across remote clusters, none of them are available, and all of them have skip_unavailable set to true, + //we end up calling merge without anything to merge, we just return an empty search response + if (searchResponses.size() == 0) { + SearchHits searchHits = new SearchHits(new SearchHit[0], new TotalHits(0L, TotalHits.Relation.EQUAL_TO), Float.NaN); + InternalSearchResponse internalSearchResponse = new InternalSearchResponse(searchHits, + InternalAggregations.EMPTY, null, null, false, null, 0); + return new SearchResponse(internalSearchResponse, null, 0, 0, 0, searchTimeProvider.buildTookInMillis(), + ShardSearchFailure.EMPTY_ARRAY, clusters); + } int totalShards = 0; int skippedShards = 0; int successfulShards = 0; @@ -115,7 +129,7 @@ SearchResponse getMergedResponse(Clusters clusters) { List failures = new ArrayList<>(); Map profileResults = new HashMap<>(); List aggs = new ArrayList<>(); - Map shards = new TreeMap<>(); + Map shards = new TreeMap<>(); List topDocsList = new ArrayList<>(searchResponses.size()); Map> groupedSuggestions = new HashMap<>(); Boolean trackTotalHits = null; @@ -171,10 +185,11 @@ SearchResponse getMergedResponse(Clusters clusters) { Suggest suggest = groupedSuggestions.isEmpty() ? null : new Suggest(Suggest.reduce(groupedSuggestions)); InternalAggregations reducedAggs = InternalAggregations.reduce(aggs, reduceContextFunction.apply(true)); ShardSearchFailure[] shardFailures = failures.toArray(ShardSearchFailure.EMPTY_ARRAY); + SearchProfileShardResults profileShardResults = profileResults.isEmpty() ? null : new SearchProfileShardResults(profileResults); //make failures ordering consistent with ordinary search and CCS Arrays.sort(shardFailures, FAILURES_COMPARATOR); - InternalSearchResponse response = new InternalSearchResponse(mergedSearchHits, reducedAggs, suggest, - new SearchProfileShardResults(profileResults), topDocsStats.timedOut, topDocsStats.terminatedEarly, numReducePhases); + InternalSearchResponse response = new InternalSearchResponse(mergedSearchHits, reducedAggs, suggest, profileShardResults, + topDocsStats.timedOut, topDocsStats.terminatedEarly, numReducePhases); long tookInMillis = searchTimeProvider.buildTookInMillis(); return new SearchResponse(response, null, totalShards, successfulShards, skippedShards, tookInMillis, shardFailures, clusters); } @@ -210,7 +225,7 @@ private ShardId extractShardId(ShardSearchFailure failure) { } }; - private static TopDocs searchHitsToTopDocs(SearchHits searchHits, TotalHits totalHits, Map shards) { + private static TopDocs searchHitsToTopDocs(SearchHits searchHits, TotalHits totalHits, Map shards) { SearchHit[] hits = searchHits.getHits(); ScoreDoc[] scoreDocs = new ScoreDoc[hits.length]; final TopDocs topDocs; @@ -228,7 +243,8 @@ private static TopDocs searchHitsToTopDocs(SearchHits searchHits, TotalHits tota for (int i = 0; i < hits.length; i++) { SearchHit hit = hits[i]; - ShardId shardId = hit.getShard().getShardId(); + SearchShardTarget shard = hit.getShard(); + ShardIdAndClusterAlias shardId = new ShardIdAndClusterAlias(shard.getShardId(), shard.getClusterAlias()); shards.putIfAbsent(shardId, null); final SortField[] sortFields = searchHits.getSortFields(); final Object[] sortValues; @@ -246,18 +262,21 @@ private static TopDocs searchHitsToTopDocs(SearchHits searchHits, TotalHits tota return topDocs; } - private static void setShardIndex(Map shards, List topDocsList) { - int shardIndex = 0; - for (Map.Entry shard : shards.entrySet()) { - shard.setValue(shardIndex++); + private static void setShardIndex(Map shards, List topDocsList) { + { + //assign a different shardIndex to each shard, based on their shardId natural ordering and their cluster alias + int shardIndex = 0; + for (Map.Entry shard : shards.entrySet()) { + shard.setValue(shardIndex++); + } } - //and go through all the scoreDocs from each cluster and set their corresponding shardIndex + //go through all the scoreDocs from each cluster and set their corresponding shardIndex for (TopDocs topDocs : topDocsList) { for (ScoreDoc scoreDoc : topDocs.scoreDocs) { FieldDocAndSearchHit fieldDocAndSearchHit = (FieldDocAndSearchHit) scoreDoc; - //When hits come from the indices with same names on multiple clusters and same shard identifier, we rely on such indices - //to have a different uuid across multiple clusters. That's how they will get a different shardIndex. - ShardId shardId = fieldDocAndSearchHit.searchHit.getShard().getShardId(); + SearchShardTarget shard = fieldDocAndSearchHit.searchHit.getShard(); + ShardIdAndClusterAlias shardId = new ShardIdAndClusterAlias(shard.getShardId(), shard.getClusterAlias()); + assert shards.containsKey(shardId); fieldDocAndSearchHit.shardIndex = shards.get(shardId); } } @@ -294,4 +313,58 @@ private static final class FieldDocAndSearchHit extends FieldDoc { this.searchHit = searchHit; } } + + /** + * This class is used instead of plain {@link ShardId} to support the scenario where the same remote cluster is registered twice using + * different aliases. In that case searching across the same cluster twice would make an assertion in lucene fail + * (see TopDocs#tieBreakLessThan line 86). Generally, indices with same names on different clusters have different index uuids which + * make their ShardIds different, which is not the case if the index is really the same one from the same cluster, in which case we + * need to look at the cluster alias and make sure to assign a different shardIndex based on that. + */ + private static final class ShardIdAndClusterAlias implements Comparable { + private final ShardId shardId; + private final String clusterAlias; + + ShardIdAndClusterAlias(ShardId shardId, String clusterAlias) { + this.shardId = shardId; + this.clusterAlias = clusterAlias; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ShardIdAndClusterAlias that = (ShardIdAndClusterAlias) o; + return shardId.equals(that.shardId) && + clusterAlias.equals(that.clusterAlias); + } + + @Override + public int hashCode() { + return Objects.hash(shardId, clusterAlias); + } + + @Override + public int compareTo(ShardIdAndClusterAlias o) { + int shardIdCompareTo = shardId.compareTo(o.shardId); + if (shardIdCompareTo != 0) { + return shardIdCompareTo; + } + int clusterAliasCompareTo = clusterAlias.compareTo(o.clusterAlias); + if (clusterAliasCompareTo != 0) { + //TODO we may want to fix this, CCS returns remote results before local ones (TransportSearchAction#mergeShardsIterators) + if (clusterAlias.equals(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY)) { + return 1; + } + if (o.clusterAlias.equals(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY)) { + return -1; + } + } + return clusterAliasCompareTo; + } + } } diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index 30e030eca7376..48ae3f1249522 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -47,8 +47,10 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchService; +import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.internal.AliasFilter; +import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.RemoteClusterAware; @@ -69,6 +71,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.LongSupplier; @@ -190,8 +193,8 @@ protected void doExecute(Task task, SearchRequest searchRequest, ActionListener< new SearchTimeProvider(searchRequest.getOrCreateAbsoluteStartMillis(), relativeStartNanos, System::nanoTime); ActionListener rewriteListener = ActionListener.wrap(source -> { if (source != searchRequest.source()) { - // only set it if it changed - we don't allow null values to be set but it might be already null be we want to catch - // situations when it possible due to a bug changes to null + // only set it if it changed - we don't allow null values to be set but it might be already null. this way we catch + // situations when source is rewritten to null due to a bug searchRequest.source(source); } final ClusterState clusterState = clusterService.state(); @@ -199,26 +202,31 @@ protected void doExecute(Task task, SearchRequest searchRequest, ActionListener< searchRequest.indices(), idx -> indexNameExpressionResolver.hasIndexOrAlias(idx, clusterState)); OriginalIndices localIndices = remoteClusterIndices.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY); if (remoteClusterIndices.isEmpty()) { - executeSearch((SearchTask)task, timeProvider, searchRequest, localIndices, Collections.emptyList(), - (clusterName, nodeId) -> null, clusterState, Collections.emptyMap(), listener, SearchResponse.Clusters.EMPTY); + executeLocalSearch(task, timeProvider, searchRequest, localIndices, clusterState, listener); } else { - AtomicInteger skippedClusters = new AtomicInteger(0); - collectSearchShards(searchRequest.indicesOptions(), searchRequest.preference(), searchRequest.routing(), skippedClusters, - remoteClusterIndices, remoteClusterService, threadPool, - ActionListener.wrap( - searchShardsResponses -> { - List remoteShardIterators = new ArrayList<>(); - Map remoteAliasFilters = new HashMap<>(); - BiFunction clusterNodeLookup = processRemoteShards( - searchShardsResponses, remoteClusterIndices, remoteShardIterators, remoteAliasFilters); - int localClusters = localIndices == null ? 0 : 1; - int totalClusters = remoteClusterIndices.size() + localClusters; - int successfulClusters = searchShardsResponses.size() + localClusters; - executeSearch((SearchTask) task, timeProvider, searchRequest, localIndices, - remoteShardIterators, clusterNodeLookup, clusterState, remoteAliasFilters, listener, - new SearchResponse.Clusters(totalClusters, successfulClusters, skippedClusters.get())); - }, - listener::onFailure)); + if (shouldMinimizeRoundtrips(searchRequest)) { + ccsRemoteReduce(searchRequest, localIndices, remoteClusterIndices, timeProvider, searchService::createReduceContext, + remoteClusterService, threadPool, listener, + (r, l) -> executeLocalSearch(task, timeProvider, r, localIndices, clusterState, l)); + } else { + AtomicInteger skippedClusters = new AtomicInteger(0); + collectSearchShards(searchRequest.indicesOptions(), searchRequest.preference(), searchRequest.routing(), + skippedClusters, remoteClusterIndices, remoteClusterService, threadPool, + ActionListener.wrap( + searchShardsResponses -> { + List remoteShardIterators = new ArrayList<>(); + Map remoteAliasFilters = new HashMap<>(); + BiFunction clusterNodeLookup = processRemoteShards( + searchShardsResponses, remoteClusterIndices, remoteShardIterators, remoteAliasFilters); + int localClusters = localIndices == null ? 0 : 1; + int totalClusters = remoteClusterIndices.size() + localClusters; + int successfulClusters = searchShardsResponses.size() + localClusters; + executeSearch((SearchTask) task, timeProvider, searchRequest, localIndices, + remoteShardIterators, clusterNodeLookup, clusterState, remoteAliasFilters, listener, + new SearchResponse.Clusters(totalClusters, successfulClusters, skippedClusters.get())); + }, + listener::onFailure)); + } } }, listener::onFailure); if (searchRequest.source() == null) { @@ -229,12 +237,79 @@ protected void doExecute(Task task, SearchRequest searchRequest, ActionListener< } } + static boolean shouldMinimizeRoundtrips(SearchRequest searchRequest) { + if (searchRequest.isCcsMinimizeRoundtrips() == false) { + return false; + } + if (searchRequest.scroll() != null) { + return false; + } + SearchSourceBuilder source = searchRequest.source(); + return source == null || source.collapse() == null || source.collapse().getInnerHits() == null || + source.collapse().getInnerHits().isEmpty(); + } + + static void ccsRemoteReduce(SearchRequest searchRequest, OriginalIndices localIndices, Map remoteIndices, + SearchTimeProvider timeProvider, Function reduceContext, + RemoteClusterService remoteClusterService, ThreadPool threadPool, ActionListener listener, + BiConsumer> localSearchConsumer) { + SearchResponseMerger searchResponseMerger = createSearchResponseMerger(searchRequest.source(), timeProvider, reduceContext); + AtomicInteger skippedClusters = new AtomicInteger(0); + final AtomicReference exceptions = new AtomicReference<>(); + int totalClusters = remoteIndices.size() + (localIndices == null ? 0 : 1); + final CountDown countDown = new CountDown(totalClusters); + for (Map.Entry entry : remoteIndices.entrySet()) { + String clusterAlias = entry.getKey(); + boolean skipUnavailable = remoteClusterService.isSkipUnavailable(clusterAlias); + OriginalIndices indices = entry.getValue(); + SearchRequest ccsSearchRequest = SearchRequest.withLocalReduction(searchRequest, indices.indices(), + clusterAlias, timeProvider.getAbsoluteStartMillis()); + ActionListener ccsListener = createCCSListener(clusterAlias, skipUnavailable, countDown, + skippedClusters, exceptions, searchResponseMerger, totalClusters, listener); + Client remoteClusterClient = remoteClusterService.getRemoteClusterClient(threadPool, clusterAlias); + remoteClusterClient.search(ccsSearchRequest, ccsListener); + } + if (localIndices != null) { + ActionListener ccsListener = createCCSListener(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY, + false, countDown, skippedClusters, exceptions, searchResponseMerger, totalClusters, listener); + //here we provide the empty string a cluster alias, which means no prefix in index name, + //but the coord node will perform non final reduce as it's not null. + SearchRequest ccsLocalSearchRequest = SearchRequest.withLocalReduction(searchRequest, localIndices.indices(), + RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY, timeProvider.getAbsoluteStartMillis()); + localSearchConsumer.accept(ccsLocalSearchRequest, ccsListener); + } + } + + static SearchResponseMerger createSearchResponseMerger(SearchSourceBuilder source, SearchTimeProvider timeProvider, + Function reduceContextFunction) { + final int from; + final int size; + final int trackTotalHitsUpTo; + if (source == null) { + from = SearchService.DEFAULT_FROM; + size = SearchService.DEFAULT_SIZE; + trackTotalHitsUpTo = SearchContext.DEFAULT_TRACK_TOTAL_HITS_UP_TO; + } else { + from = source.from() == -1 ? SearchService.DEFAULT_FROM : source.from(); + size = source.size() == -1 ? SearchService.DEFAULT_SIZE : source.size(); + trackTotalHitsUpTo = source.trackTotalHitsUpTo() == null + ? SearchContext.DEFAULT_TRACK_TOTAL_HITS_UP_TO : source.trackTotalHitsUpTo(); + //here we modify the original source so we can re-use it by setting it to each outgoing search request + source.from(0); + source.size(from + size); + //TODO when searching only against a remote cluster, we could ask directly for the final number of results and let + //the remote cluster do a final reduction, yet that is not possible as we are providing a localClusterAlias which + //will automatically make the reduction non final + } + return new SearchResponseMerger(from, size, trackTotalHitsUpTo, timeProvider, reduceContextFunction); + } + static void collectSearchShards(IndicesOptions indicesOptions, String preference, String routing, AtomicInteger skippedClusters, Map remoteIndicesByCluster, RemoteClusterService remoteClusterService, ThreadPool threadPool, ActionListener> listener) { final CountDown responsesCountDown = new CountDown(remoteIndicesByCluster.size()); final Map searchShardsResponses = new ConcurrentHashMap<>(); - final AtomicReference transportException = new AtomicReference<>(); + final AtomicReference exceptions = new AtomicReference<>(); for (Map.Entry entry : remoteIndicesByCluster.entrySet()) { final String clusterAlias = entry.getKey(); boolean skipUnavailable = remoteClusterService.isSkipUnavailable(clusterAlias); @@ -242,49 +317,53 @@ static void collectSearchShards(IndicesOptions indicesOptions, String preference final String[] indices = entry.getValue().indices(); ClusterSearchShardsRequest searchShardsRequest = new ClusterSearchShardsRequest(indices) .indicesOptions(indicesOptions).local(true).preference(preference).routing(routing); - clusterClient.admin().cluster().searchShards(searchShardsRequest, new ActionListener() { + clusterClient.admin().cluster().searchShards(searchShardsRequest, + new CCSActionListener>( + clusterAlias, skipUnavailable, responsesCountDown, skippedClusters, exceptions, listener) { @Override - public void onResponse(ClusterSearchShardsResponse response) { - searchShardsResponses.put(clusterAlias, response); - maybeFinish(); + void innerOnResponse(ClusterSearchShardsResponse clusterSearchShardsResponse) { + searchShardsResponses.put(clusterAlias, clusterSearchShardsResponse); } @Override - public void onFailure(Exception e) { - if (skipUnavailable) { - skippedClusters.incrementAndGet(); - } else { - RemoteTransportException exception = - new RemoteTransportException("error while communicating with remote cluster [" + clusterAlias + "]", e); - if (transportException.compareAndSet(null, exception) == false) { - transportException.accumulateAndGet(exception, (previous, current) -> { - current.addSuppressed(previous); - return current; - }); - } - } - maybeFinish(); - } - - private void maybeFinish() { - if (responsesCountDown.countDown()) { - RemoteTransportException exception = transportException.get(); - if (exception == null) { - listener.onResponse(searchShardsResponses); - } else { - listener.onFailure(transportException.get()); - } - } + Map createFinalResponse() { + return searchShardsResponses; } } ); } } + private static ActionListener createCCSListener(String clusterAlias, boolean skipUnavailable, CountDown countDown, + AtomicInteger skippedClusters, AtomicReference exceptions, + SearchResponseMerger searchResponseMerger, int totalClusters, + ActionListener originalListener) { + return new CCSActionListener(clusterAlias, skipUnavailable, countDown, skippedClusters, + exceptions, originalListener) { + @Override + void innerOnResponse(SearchResponse searchResponse) { + searchResponseMerger.add(searchResponse); + } + + @Override + SearchResponse createFinalResponse() { + SearchResponse.Clusters clusters = new SearchResponse.Clusters(totalClusters, searchResponseMerger.numResponses(), + skippedClusters.get()); + return searchResponseMerger.getMergedResponse(clusters); + } + }; + } + + private void executeLocalSearch(Task task, SearchTimeProvider timeProvider, SearchRequest searchRequest, OriginalIndices localIndices, + ClusterState clusterState, ActionListener listener) { + executeSearch((SearchTask)task, timeProvider, searchRequest, localIndices, Collections.emptyList(), + (clusterName, nodeId) -> null, clusterState, Collections.emptyMap(), listener, SearchResponse.Clusters.EMPTY); + } + static BiFunction processRemoteShards(Map searchShardsResponses, - Map remoteIndicesByCluster, - List remoteShardIterators, - Map aliasFilterMap) { + Map remoteIndicesByCluster, + List remoteShardIterators, + Map aliasFilterMap) { Map> clusterToNode = new HashMap<>(); for (Map.Entry entry : searchShardsResponses.entrySet()) { String clusterAlias = entry.getKey(); @@ -491,4 +570,70 @@ private static void failIfOverShardCountLimit(ClusterService clusterService, int + "] to a greater value if you really want to query that many shards at the same time."); } } + + abstract static class CCSActionListener implements ActionListener { + private final String clusterAlias; + private final boolean skipUnavailable; + private final CountDown countDown; + private final AtomicInteger skippedClusters; + private final AtomicReference exceptions; + private final ActionListener originalListener; + + CCSActionListener(String clusterAlias, boolean skipUnavailable, CountDown countDown, AtomicInteger skippedClusters, + AtomicReference exceptions, ActionListener originalListener) { + this.clusterAlias = clusterAlias; + this.skipUnavailable = skipUnavailable; + this.countDown = countDown; + this.skippedClusters = skippedClusters; + this.exceptions = exceptions; + this.originalListener = originalListener; + } + + @Override + public final void onResponse(Response response) { + innerOnResponse(response); + maybeFinish(); + } + + abstract void innerOnResponse(Response response); + + @Override + public final void onFailure(Exception e) { + if (skipUnavailable) { + skippedClusters.incrementAndGet(); + } else { + Exception exception = e; + if (RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY.equals(clusterAlias) == false) { + exception = new RemoteTransportException("error while communicating with remote cluster [" + clusterAlias + "]", e); + } + if (exceptions.compareAndSet(null, exception) == false) { + exceptions.accumulateAndGet(exception, (previous, current) -> { + current.addSuppressed(previous); + return current; + }); + } + } + maybeFinish(); + } + + private void maybeFinish() { + if (countDown.countDown()) { + Exception exception = exceptions.get(); + if (exception == null) { + FinalResponse response; + try { + response = createFinalResponse(); + } catch(Exception e) { + originalListener.onFailure(e); + return; + } + originalListener.onResponse(response); + } else { + originalListener.onFailure(exceptions.get()); + } + } + } + + abstract FinalResponse createFinalResponse(); + } } diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java index dde71ad68e17f..2de583b460f84 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java @@ -1009,7 +1009,7 @@ public List readNamedWriteableList(Class catego } /** - * Reads an enum with type E that was serialized based on the value of it's ordinal + * Reads an enum with type E that was serialized based on the value of its ordinal */ public > E readEnum(Class enumClass) throws IOException { int ordinal = readVInt(); diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java index 3031e2f2e7164..175f800a7d8cf 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java @@ -1094,7 +1094,7 @@ public void writeNamedWriteableList(List list) throws } /** - * Writes an enum with type E that by serialized it based on it's ordinal value + * Writes an enum with type E based on its ordinal value */ public > void writeEnum(E enumValue) throws IOException { writeVInt(enumValue.ordinal()); diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java index 49bebe053a3f9..05a20a0cc06b9 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java @@ -147,13 +147,14 @@ public static void parseMultiLineRequest(RestRequest request, IndicesOptions ind String[] indices = Strings.splitStringByCommaToArray(request.param("index")); String[] types = Strings.splitStringByCommaToArray(request.param("type")); String searchType = request.param("search_type"); + boolean ccsMinimizeRoundtrips = request.paramAsBoolean("ccs_minimize_roundtrips", true); String routing = request.param("routing"); final Tuple sourceTuple = request.contentOrSourceParam(); final XContent xContent = sourceTuple.v1().xContent(); final BytesReference data = sourceTuple.v2(); MultiSearchRequest.readMultiLineFormat(data, xContent, consumer, indices, indicesOptions, types, routing, - searchType, request.getXContentRegistry(), allowExplicitIndex); + searchType, ccsMinimizeRoundtrips, request.getXContentRegistry(), allowExplicitIndex); } @Override diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java index 78082dd364173..00c08a124f1e4 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java @@ -173,6 +173,7 @@ public static void parseSearchRequest(SearchRequest searchRequest, RestRequest r searchRequest.routing(request.param("routing")); searchRequest.preference(request.param("preference")); searchRequest.indicesOptions(IndicesOptions.fromRequest(request, searchRequest.indicesOptions())); + searchRequest.setCcsMinimizeRoundtrips(request.paramAsBoolean("ccs_minimize_roundtrips", true)); checkRestTotalHits(request, searchRequest); } diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index ef255c8af7ad1..a14b4a328775c 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -148,6 +148,8 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv public static final Setting MAX_OPEN_SCROLL_CONTEXT = Setting.intSetting("search.max_open_scroll_context", 500, 0, Property.Dynamic, Property.NodeScope); + public static final int DEFAULT_SIZE = 10; + public static final int DEFAULT_FROM = 0; private final ThreadPool threadPool; @@ -606,10 +608,10 @@ final SearchContext createContext(ShardSearchRequest request) throws IOException // if the from and size are still not set, default them if (context.from() == -1) { - context.from(0); + context.from(DEFAULT_FROM); } if (context.size() == -1) { - context.size(10); + context.size(DEFAULT_SIZE); } // pre process diff --git a/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java index bd12f46564bac..da22ce4c96c1a 100644 --- a/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java @@ -180,7 +180,7 @@ public void testSimpleAdd4() throws Exception { assertThat(request.requests().get(2).routing(), equalTo("123")); } - public void testResponseErrorToXContent() throws IOException { + public void testResponseErrorToXContent() { long tookInMillis = randomIntBetween(1, 1000); MultiSearchResponse response = new MultiSearchResponse( new MultiSearchResponse.Item[] { @@ -262,12 +262,12 @@ public void testMultiLineSerialization() throws IOException { parsedRequest.add(r); }; MultiSearchRequest.readMultiLineFormat(new BytesArray(originalBytes), xContentType.xContent(), - consumer, null, null, null, null, null, xContentRegistry(), true); + consumer, null, null, null, null, null, null, xContentRegistry(), true); assertEquals(originalRequest, parsedRequest); } } - public void testEqualsAndHashcode() throws IOException { + public void testEqualsAndHashcode() { checkEqualsAndHashCode(createMultiSearchRequest(), MultiSearchRequestTests::copyRequest, MultiSearchRequestTests::mutate); } @@ -282,7 +282,7 @@ private static MultiSearchRequest mutate(MultiSearchRequest searchRequest) throw return mutation; } - private static MultiSearchRequest copyRequest(MultiSearchRequest request) throws IOException { + private static MultiSearchRequest copyRequest(MultiSearchRequest request) { MultiSearchRequest copy = new MultiSearchRequest(); if (request.maxConcurrentSearchRequests() > 0) { copy.maxConcurrentSearchRequests(request.maxConcurrentSearchRequests()); @@ -294,7 +294,7 @@ private static MultiSearchRequest copyRequest(MultiSearchRequest request) throws return copy; } - private static MultiSearchRequest createMultiSearchRequest() throws IOException { + private static MultiSearchRequest createMultiSearchRequest() { int numSearchRequest = randomIntBetween(1, 128); MultiSearchRequest request = new MultiSearchRequest(); for (int j = 0; j < numSearchRequest; j++) { @@ -321,7 +321,7 @@ private static MultiSearchRequest createMultiSearchRequest() throws IOException return request; } - private static SearchRequest createSimpleSearchRequest() throws IOException { + private static SearchRequest createSimpleSearchRequest() { return randomSearchRequest(() -> { // No need to return a very complex SearchSourceBuilder here, that is tested elsewhere SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); diff --git a/server/src/test/java/org/elasticsearch/action/search/MultiSearchResponseTests.java b/server/src/test/java/org/elasticsearch/action/search/MultiSearchResponseTests.java index 4bd4406d81cca..d91a4eaf02288 100644 --- a/server/src/test/java/org/elasticsearch/action/search/MultiSearchResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/MultiSearchResponseTests.java @@ -46,8 +46,8 @@ protected MultiSearchResponse createTestInstance() { int totalShards = randomIntBetween(1, Integer.MAX_VALUE); int successfulShards = randomIntBetween(0, totalShards); int skippedShards = totalShards - successfulShards; + SearchResponse.Clusters clusters = SearchResponseTests.randomClusters(); InternalSearchResponse internalSearchResponse = InternalSearchResponse.empty(); - SearchResponse.Clusters clusters = new SearchResponse.Clusters(totalShards, successfulShards, skippedShards); SearchResponse searchResponse = new SearchResponse(internalSearchResponse, null, totalShards, successfulShards, skippedShards, tookInMillis, ShardSearchFailure.EMPTY_ARRAY, clusters); items[i] = new MultiSearchResponse.Item(searchResponse, null); @@ -60,14 +60,13 @@ private static MultiSearchResponse createTestInstanceWithFailures() { MultiSearchResponse.Item[] items = new MultiSearchResponse.Item[numItems]; for (int i = 0; i < numItems; i++) { if (randomBoolean()) { - // Creating a minimal response is OK, because SearchResponse self - // is tested elsewhere. + // Creating a minimal response is OK, because SearchResponse is tested elsewhere. long tookInMillis = randomNonNegativeLong(); int totalShards = randomIntBetween(1, Integer.MAX_VALUE); int successfulShards = randomIntBetween(0, totalShards); int skippedShards = totalShards - successfulShards; + SearchResponse.Clusters clusters = SearchResponseTests.randomClusters(); InternalSearchResponse internalSearchResponse = InternalSearchResponse.empty(); - SearchResponse.Clusters clusters = new SearchResponse.Clusters(totalShards, successfulShards, skippedShards); SearchResponse searchResponse = new SearchResponse(internalSearchResponse, null, totalShards, successfulShards, skippedShards, tookInMillis, ShardSearchFailure.EMPTY_ARRAY, clusters); items[i] = new MultiSearchResponse.Item(searchResponse, null); diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java index 91f6c0c09cd20..1d2d59c60e2ae 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java @@ -22,12 +22,12 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.ArrayUtils; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.AbstractSearchTestCase; -import org.elasticsearch.search.RandomSearchRequestGenerator; import org.elasticsearch.search.Scroll; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.rescore.QueryRescorerBuilder; @@ -48,19 +48,23 @@ public class SearchRequestTests extends AbstractSearchTestCase { @Override protected SearchRequest createSearchRequest() throws IOException { + SearchRequest request = super.createSearchRequest(); if (randomBoolean()) { - return super.createSearchRequest(); + return request; } //clusterAlias and absoluteStartMillis do not have public getters/setters hence we randomize them only in this test specifically. - SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(5, 10), randomNonNegativeLong()); - RandomSearchRequestGenerator.randomSearchRequest(searchRequest, this::createSearchSourceBuilder); - return searchRequest; + return SearchRequest.withLocalReduction(request, request.indices(), + randomAlphaOfLengthBetween(5, 10), randomNonNegativeLong()); } - public void testClusterAliasValidation() { - expectThrows(NullPointerException.class, () -> new SearchRequest(null, 0)); - expectThrows(IllegalArgumentException.class, () -> new SearchRequest("", -1)); - SearchRequest searchRequest = new SearchRequest("", 0); + public void testWithLocalReduction() { + expectThrows(NullPointerException.class, () -> SearchRequest.withLocalReduction(null, Strings.EMPTY_ARRAY, "", 0)); + SearchRequest request = new SearchRequest(); + expectThrows(NullPointerException.class, () -> SearchRequest.withLocalReduction(request, null, "", 0)); + expectThrows(NullPointerException.class, () -> SearchRequest.withLocalReduction(request, new String[]{null}, "", 0)); + expectThrows(NullPointerException.class, () -> SearchRequest.withLocalReduction(request, Strings.EMPTY_ARRAY, null, 0)); + expectThrows(IllegalArgumentException.class, () -> SearchRequest.withLocalReduction(request, Strings.EMPTY_ARRAY, "", -1)); + SearchRequest searchRequest = SearchRequest.withLocalReduction(request, Strings.EMPTY_ARRAY, "", 0); assertNull(searchRequest.validate()); } @@ -72,10 +76,15 @@ public void testSerialization() throws Exception { assertNotSame(deserializedRequest, searchRequest); } - public void testClusterAliasSerialization() throws IOException { + public void testRandomVersionSerialization() throws IOException { SearchRequest searchRequest = createSearchRequest(); Version version = VersionUtils.randomVersion(random()); SearchRequest deserializedRequest = copyWriteable(searchRequest, namedWriteableRegistry, SearchRequest::new, version); + if (version.before(Version.V_7_0_0)) { + assertTrue(deserializedRequest.isCcsMinimizeRoundtrips()); + } else { + assertEquals(searchRequest.isCcsMinimizeRoundtrips(), deserializedRequest.isCcsMinimizeRoundtrips()); + } if (version.before(Version.V_6_7_0)) { assertNull(deserializedRequest.getLocalClusterAlias()); assertAbsoluteStartMillisIsCurrentTime(deserializedRequest); @@ -93,6 +102,7 @@ public void testReadFromPre6_7_0() throws IOException { assertArrayEquals(new String[]{"index"}, searchRequest.indices()); assertNull(searchRequest.getLocalClusterAlias()); assertAbsoluteStartMillisIsCurrentTime(searchRequest); + assertTrue(searchRequest.isCcsMinimizeRoundtrips()); } } @@ -215,6 +225,7 @@ private SearchRequest mutate(SearchRequest searchRequest) { mutators.add(() -> mutation.searchType(randomValueOtherThan(searchRequest.searchType(), () -> randomFrom(SearchType.DFS_QUERY_THEN_FETCH, SearchType.QUERY_THEN_FETCH)))); mutators.add(() -> mutation.source(randomValueOtherThan(searchRequest.source(), this::createSearchSourceBuilder))); + mutators.add(() -> mutation.setCcsMinimizeRoundtrips(searchRequest.isCcsMinimizeRoundtrips() == false)); randomFrom(mutators).run(); return mutation; } diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchResponseMergerTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchResponseMergerTests.java index d02b712eaaef3..712d6a60440fe 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchResponseMergerTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchResponseMergerTests.java @@ -73,7 +73,7 @@ public class SearchResponseMergerTests extends ESTestCase { @Before public void init() { - numResponses = randomIntBetween(2, 10); + numResponses = randomIntBetween(1, 10); executorService = Executors.newFixedThreadPool(numResponses); } @@ -87,7 +87,7 @@ private void addResponse(SearchResponseMerger searchResponseMerger, SearchRespon private void awaitResponsesAdded() throws InterruptedException { executorService.shutdown(); - executorService.awaitTermination(5, TimeUnit.SECONDS); + assertTrue(executorService.awaitTermination(5, TimeUnit.SECONDS)); } public void testMergeTookInMillis() throws InterruptedException { @@ -137,6 +137,7 @@ public void testMergeShardFailures() throws InterruptedException { addResponse(merger, searchResponse); } awaitResponsesAdded(); + assertEquals(numResponses, merger.numResponses()); SearchResponse.Clusters clusters = SearchResponseTests.randomClusters(); SearchResponse mergedResponse = merger.getMergedResponse(clusters); assertSame(clusters, mergedResponse.getClusters()); @@ -170,6 +171,7 @@ public void testMergeShardFailuresNullShardId() throws InterruptedException { addResponse(merger, searchResponse); } awaitResponsesAdded(); + assertEquals(numResponses, merger.numResponses()); ShardSearchFailure[] shardFailures = merger.getMergedResponse(SearchResponse.Clusters.EMPTY).getShardFailures(); assertThat(Arrays.asList(shardFailures), containsInAnyOrder(expectedFailures.toArray(ShardSearchFailure.EMPTY_ARRAY))); } @@ -189,6 +191,7 @@ public void testMergeProfileResults() throws InterruptedException { addResponse(merger, searchResponse); } awaitResponsesAdded(); + assertEquals(numResponses, merger.numResponses()); SearchResponse.Clusters clusters = SearchResponseTests.randomClusters(); SearchResponse mergedResponse = merger.getMergedResponse(clusters); assertSame(clusters, mergedResponse.getClusters()); @@ -221,6 +224,7 @@ public void testMergeSuggestions() throws InterruptedException { addResponse(searchResponseMerger, searchResponse); } awaitResponsesAdded(); + assertEquals(numResponses, searchResponseMerger.numResponses()); SearchResponse.Clusters clusters = SearchResponseTests.randomClusters(); SearchResponse mergedResponse = searchResponseMerger.getMergedResponse(clusters); assertSame(clusters, mergedResponse.getClusters()); @@ -267,6 +271,7 @@ public void testMergeAggs() throws InterruptedException { addResponse(searchResponseMerger, searchResponse); } awaitResponsesAdded(); + assertEquals(numResponses, searchResponseMerger.numResponses()); SearchResponse.Clusters clusters = SearchResponseTests.randomClusters(); SearchResponse mergedResponse = searchResponseMerger.getMergedResponse(clusters); assertSame(clusters, mergedResponse.getClusters()); @@ -334,7 +339,7 @@ public void testMergeSearchHits() throws InterruptedException { Iterator> indicesIterator = randomRealisticIndices(numIndices, numResponses).entrySet().iterator(); for (int i = 0; i < numResponses; i++) { Map.Entry entry = indicesIterator.next(); - String clusterAlias = entry.getKey().equals(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY) ? null : entry.getKey(); + String clusterAlias = entry.getKey(); Index[] indices = entry.getValue(); int total = randomIntBetween(1, 1000); expectedTotal += total; @@ -386,7 +391,7 @@ public void testMergeSearchHits() throws InterruptedException { } awaitResponsesAdded(); - + assertEquals(numResponses, searchResponseMerger.numResponses()); final SearchResponse.Clusters clusters = SearchResponseTests.randomClusters(); SearchResponse searchResponse = searchResponseMerger.getMergedResponse(clusters); @@ -434,6 +439,33 @@ public void testMergeSearchHits() throws InterruptedException { } } + public void testMergeNoResponsesAdded() { + long currentRelativeTime = randomLong(); + final SearchTimeProvider timeProvider = new SearchTimeProvider(randomLong(), 0, () -> currentRelativeTime); + SearchResponseMerger merger = new SearchResponseMerger(0, 10, Integer.MAX_VALUE, timeProvider, flag -> null); + SearchResponse.Clusters clusters = SearchResponseTests.randomClusters(); + assertEquals(0, merger.numResponses()); + SearchResponse response = merger.getMergedResponse(clusters); + assertSame(clusters, response.getClusters()); + assertEquals(TimeUnit.NANOSECONDS.toMillis(currentRelativeTime), response.getTook().millis()); + assertEquals(0, response.getTotalShards()); + assertEquals(0, response.getSuccessfulShards()); + assertEquals(0, response.getSkippedShards()); + assertEquals(0, response.getFailedShards()); + assertEquals(0, response.getNumReducePhases()); + assertFalse(response.isTimedOut()); + assertNotNull(response.getHits().getTotalHits()); + assertEquals(0, response.getHits().getTotalHits().value); + assertEquals(0, response.getHits().getHits().length); + assertEquals(TotalHits.Relation.EQUAL_TO, response.getHits().getTotalHits().relation); + assertNull(response.getScrollId()); + assertSame(InternalAggregations.EMPTY, response.getAggregations()); + assertNull(response.getSuggest()); + assertEquals(0, response.getProfileResults().size()); + assertNull(response.isTerminatedEarly()); + assertEquals(0, response.getShardFailures().length); + } + private static Tuple randomTrackTotalHits() { switch(randomIntBetween(0, 2)) { case 0: @@ -499,8 +531,11 @@ private static Map randomRealisticIndices(int numIndices, int n for (int i = 0; i < numClusters; i++) { Index[] indices = new Index[indicesNames.length]; for (int j = 0; j < indices.length; j++) { - //Realistically clusters have the same indices with same names, but different uuid - indices[j] = new Index(indicesNames[j], randomAlphaOfLength(10)); + String indexName = indicesNames[j]; + //Realistically clusters have the same indices with same names, but different uuid. Yet it can happen that the same cluster + //is registered twice with different aliases and searched multiple times as part of the same search request. + String indexUuid = frequently() ? randomAlphaOfLength(10) : indexName; + indices[j] = new Index(indexName, indexUuid); } String clusterAlias; if (frequently() || indicesPerCluster.containsKey(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY)) { @@ -551,10 +586,22 @@ public int compare(SearchHit a, SearchHit b) { } } } - int shardIdCompareTo = a.getShard().getShardId().compareTo(b.getShard().getShardId()); + SearchShardTarget aShard = a.getShard(); + SearchShardTarget bShard = b.getShard(); + int shardIdCompareTo = aShard.getShardId().compareTo(bShard.getShardId()); if (shardIdCompareTo != 0) { return shardIdCompareTo; } + int clusterAliasCompareTo = aShard.getClusterAlias().compareTo(bShard.getClusterAlias()); + if (clusterAliasCompareTo != 0) { + if (aShard.getClusterAlias().equals(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY)) { + return 1; + } + if (bShard.getClusterAlias().equals(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY)) { + return -1; + } + return clusterAliasCompareTo; + } return Integer.compare(a.docId(), b.docId()); } } diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java index f07be38765f66..18890e1339557 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java @@ -20,13 +20,11 @@ package org.elasticsearch.action.search; import org.apache.lucene.search.TotalHits; +import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -246,7 +244,8 @@ public void testToXContent() { new InternalSearchResponse( new SearchHits(hits, new TotalHits(100, TotalHits.Relation.EQUAL_TO), 1.5f), null, null, null, false, null, 1 ), - null, 0, 0, 0, 0, ShardSearchFailure.EMPTY_ARRAY, new SearchResponse.Clusters(5, 3, 2)); + null, 0, 0, 0, 0, ShardSearchFailure.EMPTY_ARRAY, + new SearchResponse.Clusters(5, 3, 2)); StringBuilder expectedString = new StringBuilder(); expectedString.append("{"); { @@ -279,24 +278,18 @@ public void testToXContent() { public void testSerialization() throws IOException { SearchResponse searchResponse = createTestItem(false); - BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); - searchResponse.writeTo(bytesStreamOutput); - try (StreamInput in = new NamedWriteableAwareStreamInput( - StreamInput.wrap(bytesStreamOutput.bytes().toBytesRef().bytes), namedWriteableRegistry)) { - SearchResponse serialized = new SearchResponse(); - serialized.readFrom(in); - if (searchResponse.getHits().getTotalHits() == null) { - assertNull(serialized.getHits().getTotalHits()); - } else { - assertEquals(searchResponse.getHits().getTotalHits().value, serialized.getHits().getTotalHits().value); - assertEquals(searchResponse.getHits().getTotalHits().relation, serialized.getHits().getTotalHits().relation); - } - assertEquals(searchResponse.getHits().getHits().length, serialized.getHits().getHits().length); - assertEquals(searchResponse.getNumReducePhases(), serialized.getNumReducePhases()); - assertEquals(searchResponse.getFailedShards(), serialized.getFailedShards()); - assertEquals(searchResponse.getTotalShards(), serialized.getTotalShards()); - assertEquals(searchResponse.getSkippedShards(), serialized.getSkippedShards()); - assertEquals(searchResponse.getClusters(), serialized.getClusters()); + SearchResponse deserialized = copyStreamable(searchResponse, namedWriteableRegistry, SearchResponse::new, Version.CURRENT); + if (searchResponse.getHits().getTotalHits() == null) { + assertNull(deserialized.getHits().getTotalHits()); + } else { + assertEquals(searchResponse.getHits().getTotalHits().value, deserialized.getHits().getTotalHits().value); + assertEquals(searchResponse.getHits().getTotalHits().relation, deserialized.getHits().getTotalHits().relation); } + assertEquals(searchResponse.getHits().getHits().length, deserialized.getHits().getHits().length); + assertEquals(searchResponse.getNumReducePhases(), deserialized.getNumReducePhases()); + assertEquals(searchResponse.getFailedShards(), deserialized.getFailedShards()); + assertEquals(searchResponse.getTotalShards(), deserialized.getTotalShards()); + assertEquals(searchResponse.getSkippedShards(), deserialized.getSkippedShards()); + assertEquals(searchResponse.getClusters(), deserialized.getClusters()); } } diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionSingleNodeTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionSingleNodeTests.java index 19bd76ec09da2..8fd75c5fd673d 100644 --- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionSingleNodeTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionSingleNodeTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.common.Strings; import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchHit; @@ -41,7 +42,7 @@ public void testLocalClusterAlias() { assertEquals(RestStatus.CREATED, indexResponse.status()); { - SearchRequest searchRequest = new SearchRequest("local", nowInMillis); + SearchRequest searchRequest = SearchRequest.withLocalReduction(new SearchRequest(), Strings.EMPTY_ARRAY, "local", nowInMillis); SearchResponse searchResponse = client().search(searchRequest).actionGet(); assertEquals(1, searchResponse.getHits().getTotalHits().value); SearchHit[] hits = searchResponse.getHits().getHits(); @@ -52,7 +53,7 @@ public void testLocalClusterAlias() { assertEquals("1", hit.getId()); } { - SearchRequest searchRequest = new SearchRequest("", nowInMillis); + SearchRequest searchRequest = SearchRequest.withLocalReduction(new SearchRequest(), Strings.EMPTY_ARRAY, "", nowInMillis); SearchResponse searchResponse = client().search(searchRequest).actionGet(); assertEquals(1, searchResponse.getHits().getTotalHits().value); SearchHit[] hits = searchResponse.getHits().getHits(); @@ -93,19 +94,19 @@ public void testAbsoluteStartMillis() { assertEquals(0, searchResponse.getTotalShards()); } { - SearchRequest searchRequest = new SearchRequest("", 0); + SearchRequest searchRequest = SearchRequest.withLocalReduction(new SearchRequest(), Strings.EMPTY_ARRAY, "", 0); SearchResponse searchResponse = client().search(searchRequest).actionGet(); assertEquals(2, searchResponse.getHits().getTotalHits().value); } { - SearchRequest searchRequest = new SearchRequest("", 0); + SearchRequest searchRequest = SearchRequest.withLocalReduction(new SearchRequest(), Strings.EMPTY_ARRAY, "", 0); searchRequest.indices(""); SearchResponse searchResponse = client().search(searchRequest).actionGet(); assertEquals(1, searchResponse.getHits().getTotalHits().value); assertEquals("test-1970.01.01", searchResponse.getHits().getHits()[0].getIndex()); } { - SearchRequest searchRequest = new SearchRequest("", 0); + SearchRequest searchRequest = SearchRequest.withLocalReduction(new SearchRequest(), Strings.EMPTY_ARRAY, "", 0); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); RangeQueryBuilder rangeQuery = new RangeQueryBuilder("date"); rangeQuery.gte("1970-01-01"); diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java index 1b99beee65e81..8a5859e200eac 100644 --- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java @@ -19,6 +19,8 @@ package org.elasticsearch.action.search; +import org.apache.lucene.search.TotalHits; +import org.apache.lucene.util.SetOnce; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.LatchedActionListener; @@ -34,13 +36,24 @@ import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.cluster.routing.TestShardRouting; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.index.query.InnerHitBuilder; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.Scroll; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.collapse.CollapseBuilder; import org.elasticsearch.search.internal.AliasFilter; +import org.elasticsearch.search.internal.InternalSearchResponse; +import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.threadpool.TestThreadPool; @@ -320,169 +333,495 @@ public void close() { } } - private MockTransportService startTransport(String id, List knownNodes) { - return RemoteClusterConnectionTests.startTransport(id, knownNodes, Version.CURRENT, threadPool); - } - - public void testCollectSearchShards() throws Exception { - int numClusters = randomIntBetween(2, 10); + private MockTransportService[] startTransport(int numClusters, DiscoveryNode[] nodes, Map remoteIndices, + Settings.Builder settingsBuilder) { MockTransportService[] mockTransportServices = new MockTransportService[numClusters]; - DiscoveryNode[] nodes = new DiscoveryNode[numClusters]; - Map remoteIndicesByCluster = new HashMap<>(); - Settings.Builder builder = Settings.builder(); for (int i = 0; i < numClusters; i++) { List knownNodes = new CopyOnWriteArrayList<>(); - MockTransportService remoteSeedTransport = startTransport("node_remote" + i, knownNodes); + MockTransportService remoteSeedTransport = RemoteClusterConnectionTests.startTransport("node_remote" + i, knownNodes, + Version.CURRENT, threadPool); mockTransportServices[i] = remoteSeedTransport; DiscoveryNode remoteSeedNode = remoteSeedTransport.getLocalDiscoNode(); knownNodes.add(remoteSeedNode); nodes[i] = remoteSeedNode; - builder.put("cluster.remote.remote" + i + ".seeds", remoteSeedNode.getAddress().toString()); - remoteIndicesByCluster.put("remote" + i, new OriginalIndices(new String[]{"index"}, IndicesOptions.lenientExpandOpen())); + settingsBuilder.put("cluster.remote.remote" + i + ".seeds", remoteSeedNode.getAddress().toString()); + remoteIndices.put("remote" + i, new OriginalIndices(new String[]{"index"}, IndicesOptions.lenientExpandOpen())); } + return mockTransportServices; + } + + private static SearchResponse emptySearchResponse() { + InternalSearchResponse response = new InternalSearchResponse(new SearchHits(new SearchHit[0], + new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN), InternalAggregations.EMPTY, null, null, false, null, 1); + return new SearchResponse(response, null, 1, 1, 0, 100, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY); + } + + public void testCCSRemoteReduceMergeFails() throws Exception { + int numClusters = randomIntBetween(2, 10); + DiscoveryNode[] nodes = new DiscoveryNode[numClusters]; + Map remoteIndicesByCluster = new HashMap<>(); + Settings.Builder builder = Settings.builder(); + MockTransportService[] mockTransportServices = startTransport(numClusters, nodes, remoteIndicesByCluster, builder); Settings settings = builder.build(); + boolean local = randomBoolean(); + OriginalIndices localIndices = local ? new OriginalIndices(new String[]{"index"}, SearchRequest.DEFAULT_INDICES_OPTIONS) : null; + TransportSearchAction.SearchTimeProvider timeProvider = new TransportSearchAction.SearchTimeProvider(0, 0, () -> 0); + Function reduceContext = finalReduce -> null; + try (MockTransportService service = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, null)) { + service.start(); + service.acceptIncomingRequests(); + RemoteClusterService remoteClusterService = service.getRemoteClusterService(); + SearchRequest searchRequest = new SearchRequest(); + searchRequest.preference("null_target"); + final CountDownLatch latch = new CountDownLatch(1); + SetOnce>> setOnce = new SetOnce<>(); + AtomicReference failure = new AtomicReference<>(); + LatchedActionListener listener = new LatchedActionListener<>( + ActionListener.wrap(r -> fail("no response expected"), failure::set), latch); + TransportSearchAction.ccsRemoteReduce(searchRequest, localIndices, remoteIndicesByCluster, timeProvider, reduceContext, + remoteClusterService, threadPool, listener, (r, l) -> setOnce.set(Tuple.tuple(r, l))); + if (localIndices == null) { + assertNull(setOnce.get()); + } else { + Tuple> tuple = setOnce.get(); + assertEquals("", tuple.v1().getLocalClusterAlias()); + assertThat(tuple.v2(), instanceOf(TransportSearchAction.CCSActionListener.class)); + tuple.v2().onResponse(emptySearchResponse()); + } + awaitLatch(latch, 5, TimeUnit.SECONDS); + assertNotNull(failure.get()); + //the intention here is not to test that we throw NPE, rather to trigger a situation that makes + //SearchResponseMerger#getMergedResponse fail unexpectedly and verify that the listener is properly notified with the NPE + assertThat(failure.get(), instanceOf(NullPointerException.class)); + assertEquals(0, service.getConnectionManager().size()); + } finally { + for (MockTransportService mockTransportService : mockTransportServices) { + mockTransportService.close(); + } + } + } - try { - try (MockTransportService service = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, null)) { - service.start(); - service.acceptIncomingRequests(); - RemoteClusterService remoteClusterService = service.getRemoteClusterService(); - { - final CountDownLatch latch = new CountDownLatch(1); - AtomicReference> response = new AtomicReference<>(); - AtomicInteger skippedClusters = new AtomicInteger(); - TransportSearchAction.collectSearchShards(IndicesOptions.lenientExpandOpen(), null, null, skippedClusters, - remoteIndicesByCluster, remoteClusterService, threadPool, - new LatchedActionListener<>(ActionListener.wrap(response::set, e -> fail("no failures expected")), latch)); - awaitLatch(latch, 5, TimeUnit.SECONDS); - assertEquals(0, skippedClusters.get()); - assertNotNull(response.get()); - Map map = response.get(); - assertEquals(numClusters, map.size()); - for (int i = 0; i < numClusters; i++) { - String clusterAlias = "remote" + i; - assertTrue(map.containsKey(clusterAlias)); - ClusterSearchShardsResponse shardsResponse = map.get(clusterAlias); - assertEquals(1, shardsResponse.getNodes().length); - } + public void testCCSRemoteReduce() throws Exception { + int numClusters = randomIntBetween(2, 10); + DiscoveryNode[] nodes = new DiscoveryNode[numClusters]; + Map remoteIndicesByCluster = new HashMap<>(); + Settings.Builder builder = Settings.builder(); + MockTransportService[] mockTransportServices = startTransport(numClusters, nodes, remoteIndicesByCluster, builder); + Settings settings = builder.build(); + boolean local = randomBoolean(); + OriginalIndices localIndices = local ? new OriginalIndices(new String[]{"index"}, SearchRequest.DEFAULT_INDICES_OPTIONS) : null; + int totalClusters = numClusters + (local ? 1 : 0); + TransportSearchAction.SearchTimeProvider timeProvider = new TransportSearchAction.SearchTimeProvider(0, 0, () -> 0); + Function reduceContext = finalReduce -> null; + try (MockTransportService service = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, null)) { + service.start(); + service.acceptIncomingRequests(); + RemoteClusterService remoteClusterService = service.getRemoteClusterService(); + { + SearchRequest searchRequest = new SearchRequest(); + final CountDownLatch latch = new CountDownLatch(1); + SetOnce>> setOnce = new SetOnce<>(); + AtomicReference response = new AtomicReference<>(); + LatchedActionListener listener = new LatchedActionListener<>( + ActionListener.wrap(response::set, e -> fail("no failures expected")), latch); + TransportSearchAction.ccsRemoteReduce(searchRequest, localIndices, remoteIndicesByCluster, timeProvider, reduceContext, + remoteClusterService, threadPool, listener, (r, l) -> setOnce.set(Tuple.tuple(r, l))); + if (localIndices == null) { + assertNull(setOnce.get()); + } else { + Tuple> tuple = setOnce.get(); + assertEquals("", tuple.v1().getLocalClusterAlias()); + assertThat(tuple.v2(), instanceOf(TransportSearchAction.CCSActionListener.class)); + tuple.v2().onResponse(emptySearchResponse()); } - { - final CountDownLatch latch = new CountDownLatch(1); - AtomicReference failure = new AtomicReference<>(); - AtomicInteger skippedClusters = new AtomicInteger(0); - TransportSearchAction.collectSearchShards(IndicesOptions.lenientExpandOpen(), "index_not_found", null, skippedClusters, - remoteIndicesByCluster, remoteClusterService, threadPool, - new LatchedActionListener<>(ActionListener.wrap(r -> fail("no response expected"), failure::set), latch)); - awaitLatch(latch, 5, TimeUnit.SECONDS); - assertEquals(0, skippedClusters.get()); - assertNotNull(failure.get()); - assertThat(failure.get(), instanceOf(RemoteTransportException.class)); - RemoteTransportException remoteTransportException = (RemoteTransportException) failure.get(); - assertEquals(RestStatus.NOT_FOUND, remoteTransportException.status()); + awaitLatch(latch, 5, TimeUnit.SECONDS); + + SearchResponse searchResponse = response.get(); + assertEquals(0, searchResponse.getClusters().getSkipped()); + assertEquals(totalClusters, searchResponse.getClusters().getTotal()); + assertEquals(totalClusters, searchResponse.getClusters().getSuccessful()); + assertEquals(totalClusters + 1, searchResponse.getNumReducePhases()); + } + { + SearchRequest searchRequest = new SearchRequest(); + searchRequest.preference("index_not_found"); + final CountDownLatch latch = new CountDownLatch(1); + SetOnce>> setOnce = new SetOnce<>(); + AtomicReference failure = new AtomicReference<>(); + LatchedActionListener listener = new LatchedActionListener<>( + ActionListener.wrap(r -> fail("no response expected"), failure::set), latch); + TransportSearchAction.ccsRemoteReduce(searchRequest, localIndices, remoteIndicesByCluster, timeProvider, reduceContext, + remoteClusterService, threadPool, listener, (r, l) -> setOnce.set(Tuple.tuple(r, l))); + if (localIndices == null) { + assertNull(setOnce.get()); + } else { + Tuple> tuple = setOnce.get(); + assertEquals("", tuple.v1().getLocalClusterAlias()); + assertThat(tuple.v2(), instanceOf(TransportSearchAction.CCSActionListener.class)); + tuple.v2().onResponse(emptySearchResponse()); } + awaitLatch(latch, 5, TimeUnit.SECONDS); + assertNotNull(failure.get()); + assertThat(failure.get(), instanceOf(RemoteTransportException.class)); + RemoteTransportException remoteTransportException = (RemoteTransportException) failure.get(); + assertEquals(RestStatus.NOT_FOUND, remoteTransportException.status()); + } - int numDisconnectedClusters = randomIntBetween(1, numClusters); - Set disconnectedNodes = new HashSet<>(numDisconnectedClusters); - Set disconnectedNodesIndices = new HashSet<>(numDisconnectedClusters); - while (disconnectedNodes.size() < numDisconnectedClusters) { - int i = randomIntBetween(0, numClusters - 1); - if (disconnectedNodes.add(nodes[i])) { - assertTrue(disconnectedNodesIndices.add(i)); - } + int numDisconnectedClusters = randomIntBetween(1, numClusters); + Set disconnectedNodes = new HashSet<>(numDisconnectedClusters); + Set disconnectedNodesIndices = new HashSet<>(numDisconnectedClusters); + while (disconnectedNodes.size() < numDisconnectedClusters) { + int i = randomIntBetween(0, numClusters - 1); + if (disconnectedNodes.add(nodes[i])) { + assertTrue(disconnectedNodesIndices.add(i)); } + } - CountDownLatch disconnectedLatch = new CountDownLatch(numDisconnectedClusters); - RemoteClusterServiceTests.addConnectionListener(remoteClusterService, new TransportConnectionListener() { - @Override - public void onNodeDisconnected(DiscoveryNode node) { - if (disconnectedNodes.remove(node)) { - disconnectedLatch.countDown(); - } + CountDownLatch disconnectedLatch = new CountDownLatch(numDisconnectedClusters); + RemoteClusterServiceTests.addConnectionListener(remoteClusterService, new TransportConnectionListener() { + @Override + public void onNodeDisconnected(DiscoveryNode node) { + if (disconnectedNodes.remove(node)) { + disconnectedLatch.countDown(); } - }); - for (DiscoveryNode disconnectedNode : disconnectedNodes) { - service.addFailToSendNoConnectRule(disconnectedNode.getAddress()); } + }); + for (DiscoveryNode disconnectedNode : disconnectedNodes) { + service.addFailToSendNoConnectRule(disconnectedNode.getAddress()); + } - { - final CountDownLatch latch = new CountDownLatch(1); - AtomicInteger skippedClusters = new AtomicInteger(0); - AtomicReference failure = new AtomicReference<>(); - TransportSearchAction.collectSearchShards(IndicesOptions.lenientExpandOpen(), null, null, skippedClusters, - remoteIndicesByCluster, remoteClusterService, threadPool, - new LatchedActionListener<>(ActionListener.wrap(r -> fail("no response expected"), failure::set), latch)); - awaitLatch(latch, 5, TimeUnit.SECONDS); - assertEquals(0, skippedClusters.get()); - assertNotNull(failure.get()); - assertThat(failure.get(), instanceOf(RemoteTransportException.class)); - assertThat(failure.get().getMessage(), containsString("error while communicating with remote cluster [")); - assertThat(failure.get().getCause(), instanceOf(NodeDisconnectedException.class)); + { + SearchRequest searchRequest = new SearchRequest(); + final CountDownLatch latch = new CountDownLatch(1); + SetOnce>> setOnce = new SetOnce<>(); + AtomicReference failure = new AtomicReference<>(); + LatchedActionListener listener = new LatchedActionListener<>( + ActionListener.wrap(r -> fail("no response expected"), failure::set), latch); + TransportSearchAction.ccsRemoteReduce(searchRequest, localIndices, remoteIndicesByCluster, timeProvider, reduceContext, + remoteClusterService, threadPool, listener, (r, l) -> setOnce.set(Tuple.tuple(r, l))); + if (localIndices == null) { + assertNull(setOnce.get()); + } else { + Tuple> tuple = setOnce.get(); + assertEquals("", tuple.v1().getLocalClusterAlias()); + assertThat(tuple.v2(), instanceOf(TransportSearchAction.CCSActionListener.class)); + tuple.v2().onResponse(emptySearchResponse()); } + awaitLatch(latch, 5, TimeUnit.SECONDS); + assertNotNull(failure.get()); + assertThat(failure.get(), instanceOf(RemoteTransportException.class)); + RemoteTransportException remoteTransportException = (RemoteTransportException) failure.get(); + assertThat(failure.get().getMessage(), containsString("error while communicating with remote cluster [")); + assertThat(failure.get().getCause(), instanceOf(NodeDisconnectedException.class)); + } - //setting skip_unavailable to true for all the disconnected clusters will make the request succeed again - for (int i : disconnectedNodesIndices) { - RemoteClusterServiceTests.updateSkipUnavailable(remoteClusterService, "remote" + i, true); + //setting skip_unavailable to true for all the disconnected clusters will make the request succeed again + for (int i : disconnectedNodesIndices) { + RemoteClusterServiceTests.updateSkipUnavailable(remoteClusterService, "remote" + i, true); + } + + { + SearchRequest searchRequest = new SearchRequest(); + final CountDownLatch latch = new CountDownLatch(1); + SetOnce>> setOnce = new SetOnce<>(); + AtomicReference response = new AtomicReference<>(); + LatchedActionListener listener = new LatchedActionListener<>( + ActionListener.wrap(response::set, e -> fail("no failures expected")), latch); + TransportSearchAction.ccsRemoteReduce(searchRequest, localIndices, remoteIndicesByCluster, timeProvider, reduceContext, + remoteClusterService, threadPool, listener, (r, l) -> setOnce.set(Tuple.tuple(r, l))); + if (localIndices == null) { + assertNull(setOnce.get()); + } else { + Tuple> tuple = setOnce.get(); + assertEquals("", tuple.v1().getLocalClusterAlias()); + assertThat(tuple.v2(), instanceOf(TransportSearchAction.CCSActionListener.class)); + tuple.v2().onResponse(emptySearchResponse()); } + awaitLatch(latch, 5, TimeUnit.SECONDS); + + SearchResponse searchResponse = response.get(); + assertEquals(disconnectedNodesIndices.size(), searchResponse.getClusters().getSkipped()); + assertEquals(totalClusters, searchResponse.getClusters().getTotal()); + int successful = totalClusters - disconnectedNodesIndices.size(); + assertEquals(successful, searchResponse.getClusters().getSuccessful()); + assertEquals(successful == 0 ? 0 : successful + 1, searchResponse.getNumReducePhases()); + } + + //give transport service enough time to realize that the node is down, and to notify the connection listeners + //so that RemoteClusterConnection is left with no connected nodes, hence it will retry connecting next + assertTrue(disconnectedLatch.await(5, TimeUnit.SECONDS)); - { - final CountDownLatch latch = new CountDownLatch(1); - AtomicInteger skippedClusters = new AtomicInteger(0); - AtomicReference> response = new AtomicReference<>(); - TransportSearchAction.collectSearchShards(IndicesOptions.lenientExpandOpen(), null, null, skippedClusters, - remoteIndicesByCluster, remoteClusterService, threadPool, - new LatchedActionListener<>(ActionListener.wrap(response::set, e -> fail("no failures expected")), latch)); - awaitLatch(latch, 5, TimeUnit.SECONDS); - assertNotNull(response.get()); - Map map = response.get(); - assertEquals(numClusters - disconnectedNodesIndices.size(), map.size()); - assertEquals(skippedClusters.get(), disconnectedNodesIndices.size()); - for (int i = 0; i < numClusters; i++) { - String clusterAlias = "remote" + i; - if (disconnectedNodesIndices.contains(i)) { - assertFalse(map.containsKey(clusterAlias)); - } else { - assertNotNull(map.get(clusterAlias)); - } + service.clearAllRules(); + if (randomBoolean()) { + for (int i : disconnectedNodesIndices) { + if (randomBoolean()) { + RemoteClusterServiceTests.updateSkipUnavailable(remoteClusterService, "remote" + i, true); } + + } + } + { + SearchRequest searchRequest = new SearchRequest(); + final CountDownLatch latch = new CountDownLatch(1); + SetOnce>> setOnce = new SetOnce<>(); + AtomicReference response = new AtomicReference<>(); + LatchedActionListener listener = new LatchedActionListener<>( + ActionListener.wrap(response::set, e -> fail("no failures expected")), latch); + TransportSearchAction.ccsRemoteReduce(searchRequest, localIndices, remoteIndicesByCluster, timeProvider, reduceContext, + remoteClusterService, threadPool, listener, (r, l) -> setOnce.set(Tuple.tuple(r, l))); + if (localIndices == null) { + assertNull(setOnce.get()); + } else { + Tuple> tuple = setOnce.get(); + assertEquals("", tuple.v1().getLocalClusterAlias()); + assertThat(tuple.v2(), instanceOf(TransportSearchAction.CCSActionListener.class)); + tuple.v2().onResponse(emptySearchResponse()); } + awaitLatch(latch, 5, TimeUnit.SECONDS); + + SearchResponse searchResponse = response.get(); + assertEquals(0, searchResponse.getClusters().getSkipped()); + assertEquals(totalClusters, searchResponse.getClusters().getTotal()); + assertEquals(totalClusters, searchResponse.getClusters().getSuccessful()); + assertEquals(totalClusters + 1, searchResponse.getNumReducePhases()); + } + assertEquals(0, service.getConnectionManager().size()); + } finally { + for (MockTransportService mockTransportService : mockTransportServices) { + mockTransportService.close(); + } + } + } - //give transport service enough time to realize that the node is down, and to notify the connection listeners - //so that RemoteClusterConnection is left with no connected nodes, hence it will retry connecting next - assertTrue(disconnectedLatch.await(5, TimeUnit.SECONDS)); + public void testCollectSearchShards() throws Exception { + int numClusters = randomIntBetween(2, 10); + DiscoveryNode[] nodes = new DiscoveryNode[numClusters]; + Map remoteIndicesByCluster = new HashMap<>(); + Settings.Builder builder = Settings.builder(); + MockTransportService[] mockTransportServices = startTransport(numClusters, nodes, remoteIndicesByCluster, builder); + Settings settings = builder.build(); + try (MockTransportService service = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, null)) { + service.start(); + service.acceptIncomingRequests(); + RemoteClusterService remoteClusterService = service.getRemoteClusterService(); + { + final CountDownLatch latch = new CountDownLatch(1); + AtomicReference> response = new AtomicReference<>(); + AtomicInteger skippedClusters = new AtomicInteger(); + TransportSearchAction.collectSearchShards(IndicesOptions.lenientExpandOpen(), null, null, skippedClusters, + remoteIndicesByCluster, remoteClusterService, threadPool, + new LatchedActionListener<>(ActionListener.wrap(response::set, e -> fail("no failures expected")), latch)); + awaitLatch(latch, 5, TimeUnit.SECONDS); + assertEquals(0, skippedClusters.get()); + assertNotNull(response.get()); + Map map = response.get(); + assertEquals(numClusters, map.size()); + for (int i = 0; i < numClusters; i++) { + String clusterAlias = "remote" + i; + assertTrue(map.containsKey(clusterAlias)); + ClusterSearchShardsResponse shardsResponse = map.get(clusterAlias); + assertEquals(1, shardsResponse.getNodes().length); + } + } + { + final CountDownLatch latch = new CountDownLatch(1); + AtomicReference failure = new AtomicReference<>(); + AtomicInteger skippedClusters = new AtomicInteger(0); + TransportSearchAction.collectSearchShards(IndicesOptions.lenientExpandOpen(), "index_not_found", null, skippedClusters, + remoteIndicesByCluster, remoteClusterService, threadPool, + new LatchedActionListener<>(ActionListener.wrap(r -> fail("no response expected"), failure::set), latch)); + awaitLatch(latch, 5, TimeUnit.SECONDS); + assertEquals(0, skippedClusters.get()); + assertNotNull(failure.get()); + assertThat(failure.get(), instanceOf(RemoteTransportException.class)); + RemoteTransportException remoteTransportException = (RemoteTransportException) failure.get(); + assertEquals(RestStatus.NOT_FOUND, remoteTransportException.status()); + } - service.clearAllRules(); - if (randomBoolean()) { - for (int i : disconnectedNodesIndices) { - if (randomBoolean()) { - RemoteClusterServiceTests.updateSkipUnavailable(remoteClusterService, "remote" + i, true); - } + int numDisconnectedClusters = randomIntBetween(1, numClusters); + Set disconnectedNodes = new HashSet<>(numDisconnectedClusters); + Set disconnectedNodesIndices = new HashSet<>(numDisconnectedClusters); + while (disconnectedNodes.size() < numDisconnectedClusters) { + int i = randomIntBetween(0, numClusters - 1); + if (disconnectedNodes.add(nodes[i])) { + assertTrue(disconnectedNodesIndices.add(i)); + } + } + CountDownLatch disconnectedLatch = new CountDownLatch(numDisconnectedClusters); + RemoteClusterServiceTests.addConnectionListener(remoteClusterService, new TransportConnectionListener() { + @Override + public void onNodeDisconnected(DiscoveryNode node) { + if (disconnectedNodes.remove(node)) { + disconnectedLatch.countDown(); } } - { - final CountDownLatch latch = new CountDownLatch(1); - AtomicInteger skippedClusters = new AtomicInteger(0); - AtomicReference> response = new AtomicReference<>(); - TransportSearchAction.collectSearchShards(IndicesOptions.lenientExpandOpen(), null, null, skippedClusters, - remoteIndicesByCluster, remoteClusterService, threadPool, - new LatchedActionListener<>(ActionListener.wrap(response::set, e -> fail("no failures expected")), latch)); - awaitLatch(latch, 5, TimeUnit.SECONDS); - assertEquals(0, skippedClusters.get()); - assertNotNull(response.get()); - Map map = response.get(); - assertEquals(numClusters, map.size()); - for (int i = 0; i < numClusters; i++) { - String clusterAlias = "remote" + i; - assertTrue(map.containsKey(clusterAlias)); + }); + for (DiscoveryNode disconnectedNode : disconnectedNodes) { + service.addFailToSendNoConnectRule(disconnectedNode.getAddress()); + } + + { + final CountDownLatch latch = new CountDownLatch(1); + AtomicInteger skippedClusters = new AtomicInteger(0); + AtomicReference failure = new AtomicReference<>(); + TransportSearchAction.collectSearchShards(IndicesOptions.lenientExpandOpen(), null, null, skippedClusters, + remoteIndicesByCluster, remoteClusterService, threadPool, + new LatchedActionListener<>(ActionListener.wrap(r -> fail("no response expected"), failure::set), latch)); + awaitLatch(latch, 5, TimeUnit.SECONDS); + assertEquals(0, skippedClusters.get()); + assertNotNull(failure.get()); + assertThat(failure.get(), instanceOf(RemoteTransportException.class)); + assertThat(failure.get().getMessage(), containsString("error while communicating with remote cluster [")); + assertThat(failure.get().getCause(), instanceOf(NodeDisconnectedException.class)); + } + + //setting skip_unavailable to true for all the disconnected clusters will make the request succeed again + for (int i : disconnectedNodesIndices) { + RemoteClusterServiceTests.updateSkipUnavailable(remoteClusterService, "remote" + i, true); + } + + { + final CountDownLatch latch = new CountDownLatch(1); + AtomicInteger skippedClusters = new AtomicInteger(0); + AtomicReference> response = new AtomicReference<>(); + TransportSearchAction.collectSearchShards(IndicesOptions.lenientExpandOpen(), null, null, skippedClusters, + remoteIndicesByCluster, remoteClusterService, threadPool, + new LatchedActionListener<>(ActionListener.wrap(response::set, e -> fail("no failures expected")), latch)); + awaitLatch(latch, 5, TimeUnit.SECONDS); + assertNotNull(response.get()); + Map map = response.get(); + assertEquals(numClusters - disconnectedNodesIndices.size(), map.size()); + assertEquals(skippedClusters.get(), disconnectedNodesIndices.size()); + for (int i = 0; i < numClusters; i++) { + String clusterAlias = "remote" + i; + if (disconnectedNodesIndices.contains(i)) { + assertFalse(map.containsKey(clusterAlias)); + } else { assertNotNull(map.get(clusterAlias)); } } - assertEquals(0, service.getConnectionManager().size()); } + + //give transport service enough time to realize that the node is down, and to notify the connection listeners + //so that RemoteClusterConnection is left with no connected nodes, hence it will retry connecting next + assertTrue(disconnectedLatch.await(5, TimeUnit.SECONDS)); + + service.clearAllRules(); + if (randomBoolean()) { + for (int i : disconnectedNodesIndices) { + if (randomBoolean()) { + RemoteClusterServiceTests.updateSkipUnavailable(remoteClusterService, "remote" + i, true); + } + + } + } + { + final CountDownLatch latch = new CountDownLatch(1); + AtomicInteger skippedClusters = new AtomicInteger(0); + AtomicReference> response = new AtomicReference<>(); + TransportSearchAction.collectSearchShards(IndicesOptions.lenientExpandOpen(), null, null, skippedClusters, + remoteIndicesByCluster, remoteClusterService, threadPool, + new LatchedActionListener<>(ActionListener.wrap(response::set, e -> fail("no failures expected")), latch)); + awaitLatch(latch, 5, TimeUnit.SECONDS); + assertEquals(0, skippedClusters.get()); + assertNotNull(response.get()); + Map map = response.get(); + assertEquals(numClusters, map.size()); + for (int i = 0; i < numClusters; i++) { + String clusterAlias = "remote" + i; + assertTrue(map.containsKey(clusterAlias)); + assertNotNull(map.get(clusterAlias)); + } + } + assertEquals(0, service.getConnectionManager().size()); } finally { for (MockTransportService mockTransportService : mockTransportServices) { mockTransportService.close(); } } } + + public void testCreateSearchResponseMerger() { + TransportSearchAction.SearchTimeProvider timeProvider = new TransportSearchAction.SearchTimeProvider(0, 0, () -> 0); + Function reduceContext = flag -> null; + { + SearchSourceBuilder source = new SearchSourceBuilder(); + assertEquals(-1, source.size()); + assertEquals(-1, source.from()); + assertNull(source.trackTotalHitsUpTo()); + SearchResponseMerger merger = TransportSearchAction.createSearchResponseMerger(source, timeProvider, reduceContext); + assertEquals(0, merger.from); + assertEquals(10, merger.size); + assertEquals(SearchContext.DEFAULT_TRACK_TOTAL_HITS_UP_TO, merger.trackTotalHitsUpTo); + assertEquals(0, source.from()); + assertEquals(10, source.size()); + assertNull(source.trackTotalHitsUpTo()); + } + { + SearchResponseMerger merger = TransportSearchAction.createSearchResponseMerger(null, timeProvider, reduceContext); + assertEquals(0, merger.from); + assertEquals(10, merger.size); + assertEquals(SearchContext.DEFAULT_TRACK_TOTAL_HITS_UP_TO, merger.trackTotalHitsUpTo); + } + { + SearchSourceBuilder source = new SearchSourceBuilder(); + int originalFrom = randomIntBetween(0, 1000); + source.from(originalFrom); + int originalSize = randomIntBetween(0, 1000); + source.size(originalSize); + int trackTotalHitsUpTo = randomIntBetween(0, Integer.MAX_VALUE); + source.trackTotalHitsUpTo(trackTotalHitsUpTo); + SearchResponseMerger merger = TransportSearchAction.createSearchResponseMerger(source, timeProvider, reduceContext); + assertEquals(0, source.from()); + assertEquals(originalFrom + originalSize, source.size()); + assertEquals(trackTotalHitsUpTo, (int)source.trackTotalHitsUpTo()); + assertEquals(originalFrom, merger.from); + assertEquals(originalSize, merger.size); + assertEquals(trackTotalHitsUpTo, merger.trackTotalHitsUpTo); + } + } + + public void testShouldMinimizeRoundtrips() throws Exception { + { + SearchRequest searchRequest = new SearchRequest(); + assertTrue(TransportSearchAction.shouldMinimizeRoundtrips(searchRequest)); + } + { + SearchRequest searchRequest = new SearchRequest(); + searchRequest.source(new SearchSourceBuilder()); + assertTrue(TransportSearchAction.shouldMinimizeRoundtrips(searchRequest)); + } + { + SearchRequest searchRequest = new SearchRequest(); + searchRequest.scroll("5s"); + assertFalse(TransportSearchAction.shouldMinimizeRoundtrips(searchRequest)); + } + { + SearchRequest searchRequest = new SearchRequest(); + SearchSourceBuilder source = new SearchSourceBuilder(); + searchRequest.source(source); + CollapseBuilder collapseBuilder = new CollapseBuilder("field"); + source.collapse(collapseBuilder); + collapseBuilder.setInnerHits(new InnerHitBuilder("inner")); + assertFalse(TransportSearchAction.shouldMinimizeRoundtrips(searchRequest)); + } + { + SearchRequestTests searchRequestTests = new SearchRequestTests(); + searchRequestTests.setUp(); + SearchRequest searchRequest = searchRequestTests.createSearchRequest(); + searchRequest.scroll((Scroll)null); + SearchSourceBuilder source = searchRequest.source(); + if (source != null) { + CollapseBuilder collapse = source.collapse(); + if (collapse != null) { + collapse.setInnerHits(Collections.emptyList()); + } + } + searchRequest.setCcsMinimizeRoundtrips(true); + assertTrue(TransportSearchAction.shouldMinimizeRoundtrips(searchRequest)); + searchRequest.setCcsMinimizeRoundtrips(false); + assertFalse(TransportSearchAction.shouldMinimizeRoundtrips(searchRequest)); + } + } } diff --git a/server/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java b/server/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java index 430cd900660c3..948e29d5d67de 100644 --- a/server/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java +++ b/server/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java @@ -813,7 +813,7 @@ public void testInvalidEnum() throws IOException { assertEquals(0, input.available()); } - private void assertEqualityAfterSerialize(TimeValue value, int expectedSize) throws IOException { + private static void assertEqualityAfterSerialize(TimeValue value, int expectedSize) throws IOException { BytesStreamOutput out = new BytesStreamOutput(); out.writeTimeValue(value); assertEquals(expectedSize, out.size()); diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java index 3f85d927e9295..6e9c2e4eaf320 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java @@ -62,10 +62,10 @@ public void testConnectAndExecuteRequest() throws Exception { ClusterStateResponse clusterStateResponse = client.admin().cluster().prepareState().execute().get(); assertNotNull(clusterStateResponse); assertEquals("foo_bar_cluster", clusterStateResponse.getState().getClusterName().value()); - // also test a failure, there is no handler for search registered + // also test a failure, there is no handler for scroll registered ActionNotFoundTransportException ex = expectThrows(ActionNotFoundTransportException.class, - () -> client.prepareSearch().get()); - assertEquals("No handler for action [indices:data/read/search]", ex.getMessage()); + () -> client.prepareSearchScroll("").get()); + assertEquals("No handler for action [indices:data/read/scroll]", ex.getMessage()); } } } diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java index 3ec2506da244e..9eddac80a17c0 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.transport; +import org.apache.lucene.search.TotalHits; import org.apache.lucene.store.AlreadyClosedException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; @@ -29,6 +30,10 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -47,6 +52,10 @@ import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.mocksocket.MockServerSocket; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.internal.InternalSearchResponse; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.test.junit.annotations.TestLogging; @@ -130,6 +139,24 @@ public static MockTransportService startTransport( knownNodes.toArray(new DiscoveryNode[0]), Collections.emptyMap())); } }); + newService.registerRequestHandler(SearchAction.NAME, ThreadPool.Names.SAME, SearchRequest::new, + (request, channel, task) -> { + if ("index_not_found".equals(request.preference())) { + channel.sendResponse(new IndexNotFoundException("index")); + return; + } + SearchHits searchHits; + if ("null_target".equals(request.preference())) { + searchHits = new SearchHits(new SearchHit[] {new SearchHit(0)}, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1F); + } else { + searchHits = new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN); + } + InternalSearchResponse response = new InternalSearchResponse(searchHits, + InternalAggregations.EMPTY, null, null, false, null, 1); + SearchResponse searchResponse = new SearchResponse(response, null, 1, 1, 0, 100, ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY); + channel.sendResponse(searchResponse); + }); newService.registerRequestHandler(ClusterStateAction.NAME, ThreadPool.Names.SAME, ClusterStateRequest::new, (request, channel, task) -> { DiscoveryNodes.Builder builder = DiscoveryNodes.builder(); diff --git a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java index 58dbe869b5c71..df554ea42de28 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java @@ -84,18 +84,11 @@ private RandomSearchRequestGenerator() {} * {@link #randomSearchSourceBuilder(Supplier, Supplier, Supplier, Supplier, Supplier)}. */ public static SearchRequest randomSearchRequest(Supplier randomSearchSourceBuilder) { - return randomSearchRequest(new SearchRequest(), randomSearchSourceBuilder); - } - - /** - * Set random fields to the provided search request. - * - * @param searchRequest the search request - * @param randomSearchSourceBuilder builds a random {@link SearchSourceBuilder}. You can use - * {@link #randomSearchSourceBuilder(Supplier, Supplier, Supplier, Supplier, Supplier)}. - */ - public static SearchRequest randomSearchRequest(SearchRequest searchRequest, Supplier randomSearchSourceBuilder) { + SearchRequest searchRequest = new SearchRequest(); searchRequest.allowPartialSearchResults(true); + if (randomBoolean()) { + searchRequest.setCcsMinimizeRoundtrips(randomBoolean()); + } if (randomBoolean()) { searchRequest.indices(generateRandomStringArray(10, 10, false, false)); } diff --git a/x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yml b/x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yml index 1ebd18ccaa3ac..fa8172697287e 100644 --- a/x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yml +++ b/x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yml @@ -97,6 +97,9 @@ teardown: terms: field: f1.keyword + - match: {_clusters.total: 2} + - match: {_clusters.successful: 2} + - match: {_clusters.skipped: 0} - match: { _shards.total: 5 } - match: { hits.total: 6} - match: { hits.hits.0._index: "my_remote_cluster:test_index"} @@ -115,6 +118,9 @@ teardown: terms: field: f1.keyword + - match: {_clusters.total: 1} + - match: {_clusters.successful: 1} + - match: {_clusters.skipped: 0} - match: { _shards.total: 3 } - match: { hits.total: 6} - match: { hits.hits.0._index: "my_remote_cluster:test_index"} @@ -134,6 +140,9 @@ teardown: terms: field: f1.keyword + - match: {_clusters.total: 1} + - match: {_clusters.successful: 1} + - match: {_clusters.skipped: 0} - match: { _shards.total: 3 } - match: { hits.total: 6} - match: { hits.hits.0._index: "my_remote_cluster:test_index"} @@ -152,6 +161,7 @@ teardown: terms: field: f1.keyword + - is_false: _clusters - match: { _shards.total: 2 } - match: { hits.total: 5} - match: { hits.hits.0._index: "local_index"} @@ -182,6 +192,9 @@ teardown: rest_total_hits_as_int: true index: test_remote_cluster:test_index + - match: {_clusters.total: 1} + - match: {_clusters.successful: 1} + - match: {_clusters.skipped: 0} - match: { _shards.total: 3 } - match: { hits.total: 6 } - match: { hits.hits.0._index: "test_remote_cluster:test_index" } @@ -193,6 +206,9 @@ teardown: rest_total_hits_as_int: true index: "*_remote_cluster:test_ind*" + - match: {_clusters.total: 2} + - match: {_clusters.successful: 2} + - match: {_clusters.skipped: 0} - match: { _shards.total: 6 } - match: { hits.total: 12 } @@ -205,6 +221,9 @@ teardown: rest_total_hits_as_int: true index: my_remote_cluster:aliased_test_index + - match: {_clusters.total: 1} + - match: {_clusters.successful: 1} + - match: {_clusters.skipped: 0} - match: { _shards.total: 3 } - match: { hits.total: 2 } - match: { hits.hits.0._source.filter_field: 1 } @@ -219,6 +238,9 @@ teardown: rest_total_hits_as_int: true index: my_remote_cluster:secure_alias # TODO make this a wildcard once + - match: {_clusters.total: 1} + - match: {_clusters.successful: 1} + - match: {_clusters.skipped: 0} - match: { _shards.total: 2 } - match: { hits.total: 1 } - is_true: hits.hits.0._source.secure diff --git a/x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/multi_cluster/40_scroll.yml b/x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/multi_cluster/40_scroll.yml index 6875df0847d1a..0026df4978075 100644 --- a/x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/multi_cluster/40_scroll.yml +++ b/x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/multi_cluster/40_scroll.yml @@ -52,6 +52,9 @@ teardown: query: match_all: {} + - match: {_clusters.total: 1} + - match: {_clusters.successful: 1} + - match: {_clusters.skipped: 0} - set: {_scroll_id: scroll_id} - match: {hits.total: 6 } - length: {hits.hits: 4 } @@ -66,6 +69,7 @@ teardown: rest_total_hits_as_int: true body: { "scroll_id": "$scroll_id", "scroll": "1m"} + - is_false: _clusters - match: {hits.total: 6 } - length: {hits.hits: 2 } - match: {hits.hits.0._source.filter_field: 1 } @@ -100,6 +104,9 @@ teardown: query: match_all: {} + - match: {_clusters.total: 1} + - match: {_clusters.successful: 1} + - match: {_clusters.skipped: 0} - set: {_scroll_id: scroll_id} - match: {hits.total: 6 } - length: {hits.hits: 4 } diff --git a/x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/multi_cluster/60_skip_shards.yml b/x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/multi_cluster/60_skip_shards.yml index e7842db70d263..d74e82edca7f0 100644 --- a/x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/multi_cluster/60_skip_shards.yml +++ b/x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/multi_cluster/60_skip_shards.yml @@ -66,6 +66,7 @@ teardown: - do: headers: { Authorization: "Basic am9lOnMza3JpdA==" } search: + ccs_minimize_roundtrips: false rest_total_hits_as_int: true index: "skip_shards_index,my_remote_cluster:single_doc_index" pre_filter_shard_size: 1 @@ -83,6 +84,7 @@ teardown: - do: headers: { Authorization: "Basic am9lOnMza3JpdA==" } search: + ccs_minimize_roundtrips: false rest_total_hits_as_int: true index: "skip_shards_index,my_remote_cluster:single_doc_index" pre_filter_shard_size: 1 From a9b12b38f0a037cc588fefac23f5bc5e04745a52 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Thu, 31 Jan 2019 09:19:49 -0500 Subject: [PATCH 100/100] Push primary term to replication tracker (#38044) This commit pushes the primary term into the replication tracker. This is a precursor to using the primary term to resolving ordering problems for retention leases. Namely, it can be that out-of-order retention lease sync requests arrive on a replica. To resolve this, we need a tuple of (primary term, version). For this to be, the primary term needs to be accessible in the replication tracker. As the primary term is part of the replication group anyway, this change conceptually makes sense. --- .../index/seqno/ReplicationTracker.java | 29 +++++++++ .../elasticsearch/index/shard/IndexShard.java | 59 ++++++++++--------- ...ReplicationTrackerRetentionLeaseTests.java | 4 ++ .../seqno/ReplicationTrackerTestCase.java | 1 + .../index/seqno/ReplicationTrackerTests.java | 7 ++- .../index/engine/EngineTestCase.java | 1 + 6 files changed, 69 insertions(+), 32 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java b/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java index 4a614d8874aff..bc51bc7b67164 100644 --- a/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java +++ b/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java @@ -33,6 +33,7 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.AbstractIndexShardComponent; +import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ReplicationGroup; import org.elasticsearch.index.shard.ShardId; @@ -92,6 +93,12 @@ public class ReplicationTracker extends AbstractIndexShardComponent implements L */ volatile boolean primaryMode; + /** + * The current operation primary term. Management of this value is done through {@link IndexShard} and must only be done when safe. See + * {@link #setOperationPrimaryTerm(long)}. + */ + private volatile long operationPrimaryTerm; + /** * Boolean flag that indicates if a relocation handoff is in progress. A handoff is started by calling {@link #startRelocationHandoff} * and is finished by either calling {@link #completeRelocationHandoff} or {@link #abortRelocationHandoff}, depending on whether the @@ -408,6 +415,25 @@ public boolean isPrimaryMode() { return primaryMode; } + /** + * Returns the current operation primary term. + * + * @return the primary term + */ + public long getOperationPrimaryTerm() { + return operationPrimaryTerm; + } + + /** + * Sets the current operation primary term. This method should be invoked only when no other operations are possible on the shard. That + * is, either from the constructor of {@link IndexShard} or while holding all permits on the {@link IndexShard} instance. + * + * @param operationPrimaryTerm the new operation primary term + */ + public void setOperationPrimaryTerm(final long operationPrimaryTerm) { + this.operationPrimaryTerm = operationPrimaryTerm; + } + /** * Returns whether the replication tracker has relocated away to another shard copy. */ @@ -527,6 +553,7 @@ private static long inSyncCheckpointStates( * @param shardId the shard ID * @param allocationId the allocation ID * @param indexSettings the index settings + * @param operationPrimaryTerm the current primary term * @param globalCheckpoint the last known global checkpoint for this shard, or {@link SequenceNumbers#UNASSIGNED_SEQ_NO} * @param onSyncRetentionLeases a callback when a new retention lease is created or an existing retention lease expires */ @@ -534,6 +561,7 @@ public ReplicationTracker( final ShardId shardId, final String allocationId, final IndexSettings indexSettings, + final long operationPrimaryTerm, final long globalCheckpoint, final LongConsumer onGlobalCheckpointUpdated, final LongSupplier currentTimeMillisSupplier, @@ -542,6 +570,7 @@ public ReplicationTracker( assert globalCheckpoint >= SequenceNumbers.UNASSIGNED_SEQ_NO : "illegal initial global checkpoint: " + globalCheckpoint; this.shardAllocationId = allocationId; this.primaryMode = false; + this.operationPrimaryTerm = operationPrimaryTerm; this.handoffInProgress = false; this.appliedClusterStateVersion = -1L; this.checkpoints = new HashMap<>(1 + indexSettings.getNumberOfReplicas()); diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 446b21269d5a5..42822942e3adf 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -200,7 +200,6 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl protected volatile ShardRouting shardRouting; protected volatile IndexShardState state; private volatile long pendingPrimaryTerm; // see JavaDocs for getPendingPrimaryTerm - private volatile long operationPrimaryTerm; protected final AtomicReference currentEngineReference = new AtomicReference<>(); final EngineFactory engineFactory; @@ -307,17 +306,21 @@ public IndexShard( this.checkIndexOnStartup = indexSettings.getValue(IndexSettings.INDEX_CHECK_ON_STARTUP); this.translogConfig = new TranslogConfig(shardId, shardPath().resolveTranslog(), indexSettings, bigArrays); final String aId = shardRouting.allocationId().getId(); + final long primaryTerm = indexSettings.getIndexMetaData().primaryTerm(shardId.id()); + this.pendingPrimaryTerm = primaryTerm; this.globalCheckpointListeners = new GlobalCheckpointListeners(shardId, threadPool.executor(ThreadPool.Names.LISTENER), threadPool.scheduler(), logger); - this.replicationTracker = + final ReplicationTracker replicationTracker = new ReplicationTracker( shardId, aId, indexSettings, + primaryTerm, UNASSIGNED_SEQ_NO, globalCheckpointListeners::globalCheckpointUpdated, threadPool::absoluteTimeInMillis, retentionLeaseSyncer); + this.replicationTracker = replicationTracker; // the query cache is a node-level thing, however we want the most popular filters // to be computed on a per-shard basis @@ -337,8 +340,6 @@ public boolean shouldCache(Query query) { } indexShardOperationPermits = new IndexShardOperationPermits(shardId, threadPool); searcherWrapper = indexSearcherWrapper; - pendingPrimaryTerm = indexSettings.getIndexMetaData().primaryTerm(shardId.id()); - operationPrimaryTerm = pendingPrimaryTerm; refreshListeners = buildRefreshListeners(); lastSearcherAccess.set(threadPool.relativeTimeInMillis()); persistMetadata(path, indexSettings, shardRouting, null, logger); @@ -400,7 +401,7 @@ public long getPendingPrimaryTerm() { /** Returns the primary term that is currently being used to assign to operations */ public long getOperationPrimaryTerm() { - return this.operationPrimaryTerm; + return replicationTracker.getOperationPrimaryTerm(); } /** @@ -509,7 +510,7 @@ public void updateShardState(final ShardRouting newRouting, assert pendingPrimaryTerm == newPrimaryTerm : "shard term changed on primary. expected [" + newPrimaryTerm + "] but was [" + pendingPrimaryTerm + "]" + ", current routing: " + currentRouting + ", new routing: " + newRouting; - assert operationPrimaryTerm == newPrimaryTerm; + assert getOperationPrimaryTerm() == newPrimaryTerm; try { replicationTracker.activatePrimaryMode(getLocalCheckpoint()); /* @@ -705,14 +706,14 @@ public Engine.IndexResult applyIndexOperationOnPrimary(long version, VersionType boolean isRetry) throws IOException { assert versionType.validateVersionForWrites(version); - return applyIndexOperation(getEngine(), UNASSIGNED_SEQ_NO, operationPrimaryTerm, version, versionType, ifSeqNo, + return applyIndexOperation(getEngine(), UNASSIGNED_SEQ_NO, getOperationPrimaryTerm(), version, versionType, ifSeqNo, ifPrimaryTerm, autoGeneratedTimestamp, isRetry, Engine.Operation.Origin.PRIMARY, sourceToParse); } public Engine.IndexResult applyIndexOperationOnReplica(long seqNo, long version, long autoGeneratedTimeStamp, boolean isRetry, SourceToParse sourceToParse) throws IOException { - return applyIndexOperation(getEngine(), seqNo, operationPrimaryTerm, version, null, UNASSIGNED_SEQ_NO, 0, + return applyIndexOperation(getEngine(), seqNo, getOperationPrimaryTerm(), version, null, UNASSIGNED_SEQ_NO, 0, autoGeneratedTimeStamp, isRetry, Engine.Operation.Origin.REPLICA, sourceToParse); } @@ -720,8 +721,8 @@ private Engine.IndexResult applyIndexOperation(Engine engine, long seqNo, long o @Nullable VersionType versionType, long ifSeqNo, long ifPrimaryTerm, long autoGeneratedTimeStamp, boolean isRetry, Engine.Operation.Origin origin, SourceToParse sourceToParse) throws IOException { - assert opPrimaryTerm <= this.operationPrimaryTerm: "op term [ " + opPrimaryTerm + " ] > shard term [" + this.operationPrimaryTerm - + "]"; + assert opPrimaryTerm <= getOperationPrimaryTerm() + : "op term [ " + opPrimaryTerm + " ] > shard term [" + getOperationPrimaryTerm() + "]"; ensureWriteAllowed(origin); Engine.Index operation; try { @@ -784,13 +785,13 @@ private Engine.IndexResult index(Engine engine, Engine.Index index) throws IOExc } public Engine.NoOpResult markSeqNoAsNoop(long seqNo, String reason) throws IOException { - return markSeqNoAsNoop(getEngine(), seqNo, operationPrimaryTerm, reason, Engine.Operation.Origin.REPLICA); + return markSeqNoAsNoop(getEngine(), seqNo, getOperationPrimaryTerm(), reason, Engine.Operation.Origin.REPLICA); } private Engine.NoOpResult markSeqNoAsNoop(Engine engine, long seqNo, long opPrimaryTerm, String reason, Engine.Operation.Origin origin) throws IOException { - assert opPrimaryTerm <= this.operationPrimaryTerm : "op term [ " + opPrimaryTerm + " ] > shard term [" + this.operationPrimaryTerm - + "]"; + assert opPrimaryTerm <= getOperationPrimaryTerm() + : "op term [ " + opPrimaryTerm + " ] > shard term [" + getOperationPrimaryTerm() + "]"; long startTime = System.nanoTime(); ensureWriteAllowed(origin); final Engine.NoOp noOp = new Engine.NoOp(seqNo, opPrimaryTerm, origin, startTime, reason); @@ -806,31 +807,31 @@ private Engine.NoOpResult noOp(Engine engine, Engine.NoOp noOp) { } public Engine.IndexResult getFailedIndexResult(Exception e, long version) { - return new Engine.IndexResult(e, version, operationPrimaryTerm); + return new Engine.IndexResult(e, version, getOperationPrimaryTerm()); } public Engine.DeleteResult getFailedDeleteResult(Exception e, long version) { - return new Engine.DeleteResult(e, version, operationPrimaryTerm); + return new Engine.DeleteResult(e, version, getOperationPrimaryTerm()); } public Engine.DeleteResult applyDeleteOperationOnPrimary(long version, String type, String id, VersionType versionType, long ifSeqNo, long ifPrimaryTerm) throws IOException { assert versionType.validateVersionForWrites(version); - return applyDeleteOperation(getEngine(), UNASSIGNED_SEQ_NO, operationPrimaryTerm, version, type, id, versionType, + return applyDeleteOperation(getEngine(), UNASSIGNED_SEQ_NO, getOperationPrimaryTerm(), version, type, id, versionType, ifSeqNo, ifPrimaryTerm, Engine.Operation.Origin.PRIMARY); } public Engine.DeleteResult applyDeleteOperationOnReplica(long seqNo, long version, String type, String id) throws IOException { return applyDeleteOperation( - getEngine(), seqNo, operationPrimaryTerm, version, type, id, null, UNASSIGNED_SEQ_NO, 0, Engine.Operation.Origin.REPLICA); + getEngine(), seqNo, getOperationPrimaryTerm(), version, type, id, null, UNASSIGNED_SEQ_NO, 0, Engine.Operation.Origin.REPLICA); } private Engine.DeleteResult applyDeleteOperation(Engine engine, long seqNo, long opPrimaryTerm, long version, String type, String id, @Nullable VersionType versionType, long ifSeqNo, long ifPrimaryTerm, Engine.Operation.Origin origin) throws IOException { - assert opPrimaryTerm <= this.operationPrimaryTerm : "op term [ " + opPrimaryTerm + " ] > shard term [" + this.operationPrimaryTerm - + "]"; + assert opPrimaryTerm <= getOperationPrimaryTerm() + : "op term [ " + opPrimaryTerm + " ] > shard term [" + getOperationPrimaryTerm() + "]"; ensureWriteAllowed(origin); // When there is a single type, the unique identifier is only composed of the _id, // so there is no way to differentiate foo#1 from bar#1. This is especially an issue @@ -846,7 +847,7 @@ private Engine.DeleteResult applyDeleteOperation(Engine engine, long seqNo, long return new Engine.DeleteResult(update); } } catch (MapperParsingException | IllegalArgumentException | TypeMissingException e) { - return new Engine.DeleteResult(e, version, operationPrimaryTerm, seqNo, false); + return new Engine.DeleteResult(e, version, getOperationPrimaryTerm(), seqNo, false); } if (mapperService.resolveDocumentType(type).equals(mapperService.documentMapper().type()) == false) { // We should never get there due to the fact that we generate mapping updates on deletes, @@ -1273,7 +1274,7 @@ public void prepareForIndexRecovery() { } public void trimOperationOfPreviousPrimaryTerms(long aboveSeqNo) { - getEngine().trimOperationsFromTranslog(operationPrimaryTerm, aboveSeqNo); + getEngine().trimOperationsFromTranslog(getOperationPrimaryTerm(), aboveSeqNo); } /** @@ -2388,7 +2389,7 @@ private EngineConfig newEngineConfig() { Collections.singletonList(refreshListeners), Collections.singletonList(new RefreshMetricUpdater(refreshMetric)), indexSort, circuitBreakerService, replicationTracker, replicationTracker::getRetentionLeases, - () -> operationPrimaryTerm, tombstoneDocSupplier()); + () -> getOperationPrimaryTerm(), tombstoneDocSupplier()); } /** @@ -2468,7 +2469,7 @@ private void bumpPrimaryTerm(final long newPrimaryTerm, @Nullable ActionListener combineWithAction) { assert Thread.holdsLock(mutex); assert newPrimaryTerm > pendingPrimaryTerm || (newPrimaryTerm >= pendingPrimaryTerm && combineWithAction != null); - assert operationPrimaryTerm <= pendingPrimaryTerm; + assert getOperationPrimaryTerm() <= pendingPrimaryTerm; final CountDownLatch termUpdated = new CountDownLatch(1); asyncBlockOperations(new ActionListener() { @Override @@ -2494,12 +2495,12 @@ private void innerFail(final Exception e) { public void onResponse(final Releasable releasable) { final RunOnce releaseOnce = new RunOnce(releasable::close); try { - assert operationPrimaryTerm <= pendingPrimaryTerm; + assert getOperationPrimaryTerm() <= pendingPrimaryTerm; termUpdated.await(); // indexShardOperationPermits doesn't guarantee that async submissions are executed // in the order submitted. We need to guard against another term bump - if (operationPrimaryTerm < newPrimaryTerm) { - operationPrimaryTerm = newPrimaryTerm; + if (getOperationPrimaryTerm() < newPrimaryTerm) { + replicationTracker.setOperationPrimaryTerm(newPrimaryTerm); onBlocked.run(); } } catch (final Exception e) { @@ -2585,14 +2586,14 @@ private void innerAcquireReplicaOperationPermit(final long opPrimaryTerm, final ActionListener operationListener = new ActionListener() { @Override public void onResponse(final Releasable releasable) { - if (opPrimaryTerm < operationPrimaryTerm) { + if (opPrimaryTerm < getOperationPrimaryTerm()) { releasable.close(); final String message = String.format( Locale.ROOT, "%s operation primary term [%d] is too old (current [%d])", shardId, opPrimaryTerm, - operationPrimaryTerm); + getOperationPrimaryTerm()); onPermitAcquired.onFailure(new IllegalStateException(message)); } else { assert assertReplicationTarget(); @@ -2653,7 +2654,7 @@ public void onFailure(final Exception e) { } private boolean requirePrimaryTermUpdate(final long opPrimaryTerm, final boolean allPermits) { - return (opPrimaryTerm > pendingPrimaryTerm) || (allPermits && opPrimaryTerm > operationPrimaryTerm); + return (opPrimaryTerm > pendingPrimaryTerm) || (allPermits && opPrimaryTerm > getOperationPrimaryTerm()); } public int getActiveOperationsCount() { diff --git a/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java b/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java index 90eb162374469..d7f135ffe4816 100644 --- a/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java +++ b/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java @@ -53,6 +53,7 @@ public void testAddOrRenewRetentionLease() { new ShardId("test", "_na", 0), allocationId.getId(), IndexSettingsModule.newIndexSettings("test", Settings.EMPTY), + randomNonNegativeLong(), UNASSIGNED_SEQ_NO, value -> {}, () -> 0L, @@ -88,6 +89,7 @@ public void testAddRetentionLeaseCausesRetentionLeaseSync() { new ShardId("test", "_na", 0), allocationId.getId(), IndexSettingsModule.newIndexSettings("test", Settings.EMPTY), + randomNonNegativeLong(), UNASSIGNED_SEQ_NO, value -> {}, () -> 0L, @@ -143,6 +145,7 @@ private void runExpirationTest(final boolean primaryMode) { new ShardId("test", "_na", 0), allocationId.getId(), IndexSettingsModule.newIndexSettings("test", settings), + randomNonNegativeLong(), UNASSIGNED_SEQ_NO, value -> {}, currentTimeMillis::get, @@ -215,6 +218,7 @@ public void testRetentionLeaseExpirationCausesRetentionLeaseSync() { new ShardId("test", "_na", 0), allocationId.getId(), IndexSettingsModule.newIndexSettings("test", settings), + randomNonNegativeLong(), UNASSIGNED_SEQ_NO, value -> {}, currentTimeMillis::get, diff --git a/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerTestCase.java b/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerTestCase.java index a36006a5fc4c1..5165f2e8dc9e4 100644 --- a/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerTestCase.java @@ -45,6 +45,7 @@ ReplicationTracker newTracker( new ShardId("test", "_na_", 0), allocationId.getId(), IndexSettingsModule.newIndexSettings("test", Settings.EMPTY), + randomNonNegativeLong(), UNASSIGNED_SEQ_NO, updatedGlobalCheckpoint, currentTimeMillisSupplier, diff --git a/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerTests.java b/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerTests.java index b61e3f647b9d2..7731f3cbf1d5f 100644 --- a/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerTests.java +++ b/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerTests.java @@ -683,15 +683,16 @@ public void testPrimaryContextHandoff() throws IOException { final ShardId shardId = new ShardId("test", "_na_", 0); FakeClusterState clusterState = initialState(); - final AllocationId primaryAllocationId = clusterState.routingTable.primaryShard().allocationId(); + final AllocationId aId = clusterState.routingTable.primaryShard().allocationId(); final LongConsumer onUpdate = updatedGlobalCheckpoint -> {}; + final long primaryTerm = randomNonNegativeLong(); final long globalCheckpoint = UNASSIGNED_SEQ_NO; final BiConsumer, ActionListener> onNewRetentionLease = (leases, listener) -> {}; ReplicationTracker oldPrimary = new ReplicationTracker( - shardId, primaryAllocationId.getId(), indexSettings, globalCheckpoint, onUpdate, () -> 0L, onNewRetentionLease); + shardId, aId.getId(), indexSettings, primaryTerm, globalCheckpoint, onUpdate, () -> 0L, onNewRetentionLease); ReplicationTracker newPrimary = new ReplicationTracker( - shardId, primaryAllocationId.getRelocationId(), indexSettings, globalCheckpoint, onUpdate, () -> 0L, onNewRetentionLease); + shardId, aId.getRelocationId(), indexSettings, primaryTerm, globalCheckpoint, onUpdate, () -> 0L, onNewRetentionLease); Set allocationIds = new HashSet<>(Arrays.asList(oldPrimary.shardAllocationId, newPrimary.shardAllocationId)); diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java index 35667b0f87a1c..d893168b08205 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java @@ -655,6 +655,7 @@ public EngineConfig config( shardId, allocationId.getId(), indexSettings, + randomNonNegativeLong(), SequenceNumbers.NO_OPS_PERFORMED, update -> {}, () -> 0L,