Skip to content

Commit

Permalink
Merge branch 'master' into global-checkpoint-sync
Browse files Browse the repository at this point in the history
* master: (67 commits)
  Restoring from snapshot should force generation of a new history uuid (elastic#26694)
  test: Use a single primary shard so that the exception can caught in the same way
  Move pre-6.0 node checkpoint to SequenceNumbers
  Invalid JSON request body caused endless loop (elastic#26680)
  added comment
  fix line length violation
  Moved the check to fetch phase. This basically means that we throw a better error message instead of an AOBE and not adding more restrictions.
  inner hits: Do not allow inner hits that use _source and have a non nested object field as parent
  Separate Painless Whitelist Loading from the Painless Definition (elastic#26540)
  convert more admin requests to writeable (elastic#26566)
  Handle release of 5.6.1
  Allow `InputStreamStreamInput` array size validation where applicable (elastic#26692)
  Update global checkpoint with permit after recovery
  Filter pre-6.0 nodes for checkpoint invariants
  Skip bad request REST test on pre-6.0
  Reenable BWC tests after disabling for backport
  Add global checkpoint tracking on the primary
  [Test] Fix reference/cat/allocation/line_8 test failure
  [Docs] improved description for fs.total.available_in_bytes (elastic#26657)
  Fix discovery-file plugin to use custom config path
  ...
  • Loading branch information
jasontedor committed Sep 19, 2017
2 parents e1dc814 + 04385a9 commit 86ddf79
Show file tree
Hide file tree
Showing 333 changed files with 6,373 additions and 3,904 deletions.
49 changes: 30 additions & 19 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -204,25 +204,15 @@ task branchConsistency {
}

subprojects {
project.afterEvaluate {
// ignore missing javadocs
tasks.withType(Javadoc) { Javadoc javadoc ->
// the -quiet here is because of a bug in gradle, in that adding a string option
// by itself is not added to the options. By adding quiet, both this option and
// the "value" -quiet is added, separated by a space. This is ok since the javadoc
// command already adds -quiet, so we are just duplicating it
// see https://discuss.gradle.org/t/add-custom-javadoc-option-that-does-not-take-an-argument/5959
javadoc.options.encoding='UTF8'
javadoc.options.addStringOption('Xdoclint:all,-missing', '-quiet')
/*
TODO: building javadocs with java 9 b118 is currently broken with weird errors, so
for now this is commented out...try again with the next ea build...
javadoc.executable = new File(project.javaHome, 'bin/javadoc')
if (project.javaVersion == JavaVersion.VERSION_1_9) {
// TODO: remove this hack! gradle should be passing this...
javadoc.options.addStringOption('source', '8')
}*/
}
// ignore missing javadocs
tasks.withType(Javadoc) { Javadoc javadoc ->
// the -quiet here is because of a bug in gradle, in that adding a string option
// by itself is not added to the options. By adding quiet, both this option and
// the "value" -quiet is added, separated by a space. This is ok since the javadoc
// command already adds -quiet, so we are just duplicating it
// see https://discuss.gradle.org/t/add-custom-javadoc-option-that-does-not-take-an-argument/5959
javadoc.options.encoding='UTF8'
javadoc.options.addStringOption('Xdoclint:all,-missing', '-quiet')
}

/* Sets up the dependencies that we build as part of this project but
Expand Down Expand Up @@ -280,6 +270,27 @@ subprojects {
}
}
}

// Handle javadoc dependencies across projects. Order matters: the linksOffline for
// org.elasticsearch:elasticsearch must be the last one or all the links for the
// other packages (e.g org.elasticsearch.client) will point to core rather than
// their own artifacts.
if (project.plugins.hasPlugin(BuildPlugin)) {
String artifactsHost = VersionProperties.elasticsearch.endsWith("-SNAPSHOT") ? "https://snapshots.elastic.co" : "https://artifacts.elastic.co"
Closure sortClosure = { a, b -> b.group <=> a.group }
Closure depJavadocClosure = { dep ->
if (dep.group != null && dep.group.startsWith('org.elasticsearch')) {
String substitution = project.ext.projectSubstitutions.get("${dep.group}:${dep.name}:${dep.version}")
if (substitution != null) {
project.javadoc.dependsOn substitution + ':javadoc'
String artifactPath = dep.group.replaceAll('\\.', '/') + '/' + dep.name.replaceAll('\\.', '/') + '/' + dep.version
project.javadoc.options.linksOffline artifactsHost + "/javadoc/" + artifactPath, "${project.project(substitution).buildDir}/docs/javadoc/"
}
}
}
project.configurations.compile.dependencies.findAll().toSorted(sortClosure).each(depJavadocClosure)
project.configurations.provided.dependencies.findAll().toSorted(sortClosure).each(depJavadocClosure)
}
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -455,28 +455,8 @@ class BuildPlugin implements Plugin<Project> {
}

static void configureJavadoc(Project project) {
String artifactsHost = VersionProperties.elasticsearch.endsWith("-SNAPSHOT") ? "https://snapshots.elastic.co" : "https://artifacts.elastic.co"
project.afterEvaluate {
project.tasks.withType(Javadoc) {
executable = new File(project.javaHome, 'bin/javadoc')
}
/*
* Order matters, the linksOffline for org.elasticsearch:elasticsearch must be the last one
* or all the links for the other packages (e.g org.elasticsearch.client) will point to core rather than their own artifacts
*/
Closure sortClosure = { a, b -> b.group <=> a.group }
Closure depJavadocClosure = { dep ->
if (dep.group != null && dep.group.startsWith('org.elasticsearch')) {
String substitution = project.ext.projectSubstitutions.get("${dep.group}:${dep.name}:${dep.version}")
if (substitution != null) {
project.javadoc.dependsOn substitution + ':javadoc'
String artifactPath = dep.group.replaceAll('\\.', '/') + '/' + dep.name.replaceAll('\\.', '/') + '/' + dep.version
project.javadoc.options.linksOffline artifactsHost + "/javadoc/" + artifactPath, "${project.project(substitution).buildDir}/docs/javadoc/"
}
}
}
project.configurations.compile.dependencies.findAll().toSorted(sortClosure).each(depJavadocClosure)
project.configurations.provided.dependencies.findAll().toSorted(sortClosure).each(depJavadocClosure)
project.tasks.withType(Javadoc) {
executable = new File(project.javaHome, 'bin/javadoc')
}
configureJavadocJar(project)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,11 +63,6 @@ public class ForbiddenPatternsTask extends DefaultTask {
patterns.put('nocommit should be all lowercase or all uppercase',
/((?i)nocommit)(?<!(nocommit|NOCOMMIT))/)
patterns.put('tab', /\t/)
if (System.getProperty('build.snapshot', 'true').equals('false')) {
patterns.put('norelease', /norelease|NORELEASE/)
}
patterns.put('norelease should be all lowercase or all uppercase',
/((?i)norelease)(?<!(norelease|NORELEASE))/)


inputs.property("excludes", filesFilter.excludes)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -489,6 +489,7 @@ class ClusterFormationTasks {
}
Copy installModule = project.tasks.create(name, Copy.class)
installModule.dependsOn(setup)
installModule.dependsOn(module.tasks.bundlePlugin)
installModule.into(new File(node.homeDir, "modules/${module.name}"))
installModule.from({ project.zipTree(module.tasks.bundlePlugin.outputs.files.singleFile) })
return installModule
Expand Down
6 changes: 0 additions & 6 deletions buildSrc/src/main/resources/checkstyle_suppressions.xml
Original file line number Diff line number Diff line change
Expand Up @@ -596,7 +596,6 @@
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]PathMapperTests.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]RoutingFieldMapperTests.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]SourceFieldMapperTests.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]TokenCountFieldMapperIntegrationIT.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]UpdateMappingOnClusterIT.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]UpdateMappingTests.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]BoolQueryBuilderTests.java" checks="LineLength" />
Expand Down Expand Up @@ -740,11 +739,6 @@
<suppress files="plugins[/\\]discovery-ec2[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]ec2[/\\]AmazonEC2Mock.java" checks="LineLength" />
<suppress files="plugins[/\\]discovery-gce[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]gce[/\\]GceNetworkTests.java" checks="LineLength" />
<suppress files="plugins[/\\]mapper-murmur3[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]murmur3[/\\]Murmur3FieldMapper.java" checks="LineLength" />
<suppress files="plugins[/\\]repository-azure[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]azure[/\\]blobstore[/\\]AzureBlobContainer.java" checks="LineLength" />
<suppress files="plugins[/\\]repository-azure[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]azure[/\\]blobstore[/\\]AzureBlobStore.java" checks="LineLength" />
<suppress files="plugins[/\\]repository-azure[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]azure[/\\]storage[/\\]AzureStorageServiceImpl.java" checks="LineLength" />
<suppress files="plugins[/\\]repository-azure[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]azure[/\\]storage[/\\]AzureStorageSettings.java" checks="LineLength" />
<suppress files="plugins[/\\]repository-azure[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]azure[/\\]storage[/\\]AzureStorageServiceMock.java" checks="LineLength" />
<suppress files="plugins[/\\]repository-hdfs[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]hdfs[/\\]HdfsRepository.java" checks="LineLength" />
<suppress files="plugins[/\\]repository-hdfs[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]hdfs[/\\]HdfsTests.java" checks="LineLength" />
<suppress files="plugins[/\\]repository-s3[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]s3[/\\]S3Repository.java" checks="LineLength" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.apache.http.util.EntityUtils;

import java.io.IOException;
import java.util.Locale;

/**
* Exception thrown when an elasticsearch node responds to a request with a status code that indicates an error.
Expand All @@ -39,8 +40,13 @@ public ResponseException(Response response) throws IOException {
}

private static String buildMessage(Response response) throws IOException {
String message = response.getRequestLine().getMethod() + " " + response.getHost() + response.getRequestLine().getUri()
+ ": " + response.getStatusLine().toString();
String message = String.format(Locale.ROOT,
"method [%s], host [%s], URI [%s], status line [%s]",
response.getRequestLine().getMethod(),
response.getHost(),
response.getRequestLine().getUri(),
response.getStatusLine().toString()
);

HttpEntity entity = response.getEntity();
if (entity != null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Locale;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
Expand Down Expand Up @@ -74,8 +75,14 @@ public void testResponseException() throws IOException {
assertNull(responseException.getResponse().getEntity());
}

String message = response.getRequestLine().getMethod() + " " + response.getHost() + response.getRequestLine().getUri()
+ ": " + response.getStatusLine().toString();
String message = String.format(Locale.ROOT,
"method [%s], host [%s], URI [%s], status line [%s]",
response.getRequestLine().getMethod(),
response.getHost(),
response.getRequestLine().getUri(),
response.getStatusLine().toString()
);

if (hasBody) {
message += "\n" + responseBody;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@

import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.startsWith;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.fail;
Expand Down Expand Up @@ -128,7 +129,9 @@ public void testSniffNodes() throws IOException {
} catch(ResponseException e) {
Response response = e.getResponse();
if (sniffResponse.isFailure) {
assertThat(e.getMessage(), containsString("GET " + httpHost + "/_nodes/http?timeout=" + sniffRequestTimeout + "ms"));
final String errorPrefix = "method [GET], host [" + httpHost + "], URI [/_nodes/http?timeout=" + sniffRequestTimeout
+ "ms], status line [HTTP/1.1";
assertThat(e.getMessage(), startsWith(errorPrefix));
assertThat(e.getMessage(), containsString(Integer.toString(sniffResponse.nodesInfoResponseCode)));
assertThat(response.getHost(), equalTo(httpHost));
assertThat(response.getStatusLine().getStatusCode(), equalTo(sniffResponse.nodesInfoResponseCode));
Expand Down
10 changes: 10 additions & 0 deletions core/src/main/java/org/elasticsearch/Version.java
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,12 @@ public class Version implements Comparable<Version> {
public static final Version V_5_5_3 = new Version(V_5_5_3_ID, org.apache.lucene.util.Version.LUCENE_6_6_0);
public static final int V_5_6_0_ID = 5060099;
public static final Version V_5_6_0 = new Version(V_5_6_0_ID, org.apache.lucene.util.Version.LUCENE_6_6_0);
public static final int V_5_6_1_ID = 5060199;
// TODO use proper Lucene constant once we are on a Lucene snapshot that knows about 6.6.1
public static final Version V_5_6_1 = new Version(V_5_6_1_ID, org.apache.lucene.util.Version.fromBits(6, 6, 1));
public static final int V_5_6_2_ID = 5060299;
// TODO use proper Lucene constant once we are on a Lucene snapshot that knows about 6.6.1
public static final Version V_5_6_2 = new Version(V_5_6_2_ID, org.apache.lucene.util.Version.fromBits(6, 6, 1));
public static final int V_6_0_0_alpha1_ID = 6000001;
public static final Version V_6_0_0_alpha1 =
new Version(V_6_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_0_0);
Expand Down Expand Up @@ -142,6 +148,10 @@ public static Version fromId(int id) {
return V_6_0_0_alpha2;
case V_6_0_0_alpha1_ID:
return V_6_0_0_alpha1;
case V_5_6_2_ID:
return V_5_6_2;
case V_5_6_1_ID:
return V_5_6_1;
case V_5_6_0_ID:
return V_5_6_0;
case V_5_5_3_ID:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ public abstract class DocWriteResponse extends ReplicationResponse implements Wr
private static final String FORCED_REFRESH = "forced_refresh";

/**
* An enum that represents the the results of CRUD operations, primarily used to communicate the type of
* An enum that represents the results of CRUD operations, primarily used to communicate the type of
* operation that occurred.
*/
public enum Result implements Writeable {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ private void buildTaskGroups() {
// we found parent in the list of tasks - add it to the parent list
parentTask.addGroup(taskGroup);
} else {
// we got zombie or the parent was filtered out - add it to the the top task list
// we got zombie or the parent was filtered out - add it to the top task list
topLevelTasks.add(taskGroup);
}
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,17 @@ public GetRepositoriesRequest(String[] repositories) {
this.repositories = repositories;
}

public GetRepositoriesRequest(StreamInput in) throws IOException {
super(in);
repositories = in.readStringArray();
}

@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeStringArray(repositories);
}

@Override
public ActionRequestValidationException validate() {
ActionRequestValidationException validationException = null;
Expand Down Expand Up @@ -85,13 +96,6 @@ public GetRepositoriesRequest repositories(String[] repositories) {

@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
repositories = in.readStringArray();
}

@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeStringArray(repositories);
throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ public class TransportGetRepositoriesAction extends TransportMasterNodeReadActio
@Inject
public TransportGetRepositoriesAction(Settings settings, TransportService transportService, ClusterService clusterService,
ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
super(settings, GetRepositoriesAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, GetRepositoriesRequest::new);
super(settings, GetRepositoriesAction.NAME, transportService, clusterService, threadPool, actionFilters, GetRepositoriesRequest::new, indexNameExpressionResolver);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,42 @@ public ClusterSearchShardsRequest(String... indices) {
indices(indices);
}

public ClusterSearchShardsRequest(StreamInput in) throws IOException {
super(in);
indices = new String[in.readVInt()];
for (int i = 0; i < indices.length; i++) {
indices[i] = in.readString();
}

routing = in.readOptionalString();
preference = in.readOptionalString();

if (in.getVersion().onOrBefore(Version.V_5_1_1)) {
//types
in.readStringArray();
}
indicesOptions = IndicesOptions.readIndicesOptions(in);
}

@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);

out.writeVInt(indices.length);
for (String index : indices) {
out.writeString(index);
}

out.writeOptionalString(routing);
out.writeOptionalString(preference);

if (out.getVersion().onOrBefore(Version.V_5_1_1)) {
//types
out.writeStringArray(Strings.EMPTY_ARRAY);
}
indicesOptions.writeIndicesOptions(out);
}

@Override
public ActionRequestValidationException validate() {
return null;
Expand Down Expand Up @@ -124,40 +160,6 @@ public String preference() {

@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);

indices = new String[in.readVInt()];
for (int i = 0; i < indices.length; i++) {
indices[i] = in.readString();
}

routing = in.readOptionalString();
preference = in.readOptionalString();

if (in.getVersion().onOrBefore(Version.V_5_1_1)) {
//types
in.readStringArray();
}
indicesOptions = IndicesOptions.readIndicesOptions(in);
}

@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);

out.writeVInt(indices.length);
for (String index : indices) {
out.writeString(index);
}

out.writeOptionalString(routing);
out.writeOptionalString(preference);

if (out.getVersion().onOrBefore(Version.V_5_1_1)) {
//types
out.writeStringArray(Strings.EMPTY_ARRAY);
}
indicesOptions.writeIndicesOptions(out);
throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ public TransportClusterSearchShardsAction(Settings settings, TransportService tr
IndicesService indicesService, ThreadPool threadPool, ActionFilters actionFilters,
IndexNameExpressionResolver indexNameExpressionResolver) {
super(settings, ClusterSearchShardsAction.NAME, transportService, clusterService, threadPool, actionFilters,
indexNameExpressionResolver, ClusterSearchShardsRequest::new);
ClusterSearchShardsRequest::new, indexNameExpressionResolver);
this.indicesService = indicesService;
}

Expand Down
Loading

0 comments on commit 86ddf79

Please sign in to comment.