From a382ea2db2f41d409f6ae628e6f1d4e599323980 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Tue, 16 Apr 2019 13:09:35 -0500 Subject: [PATCH 1/3] Backport FullClusterRestart tests for 7.0 This commit is copy of the applicable chnages pulled from the 7.x branch --- .../upgrades/FullClusterRestartIT.java | 322 +++++++++--------- .../upgrades/QueryBuilderBWCIT.java | 12 +- .../AbstractFullClusterRestartTestCase.java | 8 + 3 files changed, 182 insertions(+), 160 deletions(-) diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 767aade634c55..a4174d661ed1d 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -27,6 +27,7 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.client.WarningFailureException; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaDataIndexStateService; import org.elasticsearch.common.Booleans; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.Strings; @@ -36,8 +37,10 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.rest.action.document.RestBulkAction; import org.elasticsearch.rest.action.document.RestGetAction; +import org.elasticsearch.rest.action.document.RestIndexAction; import org.elasticsearch.rest.action.document.RestUpdateAction; import org.elasticsearch.rest.action.search.RestExplainAction; + import org.elasticsearch.test.NotEqualMessageBuilder; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.yaml.ObjectPath; @@ -46,6 +49,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Base64; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -61,12 +65,14 @@ import static org.elasticsearch.cluster.routing.UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING; import static org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.startsWith; /** @@ -78,15 +84,20 @@ */ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase { private final boolean supportsLenientBooleans = getOldClusterVersion().before(Version.V_6_0_0_alpha1); - private static final Version VERSION_5_1_0_UNRELEASED = Version.fromString("5.1.0"); private String index; + private String type; @Before public void setIndex() throws IOException { index = getTestName().toLowerCase(Locale.ROOT); } + @Before + public void setType() { + type = getOldClusterVersion().before(Version.V_6_7_0) ? "doc" : "_doc"; + } + public void testSearch() throws Exception { int count; if (isRunningAgainstOldCluster()) { @@ -100,7 +111,9 @@ public void testSearch() throws Exception { } { mappingsAndSettings.startObject("mappings"); - mappingsAndSettings.startObject("doc"); + if (isRunningAgainstAncientCluster()) { + mappingsAndSettings.startObject(type); + } mappingsAndSettings.startObject("properties"); { mappingsAndSettings.startObject("string"); @@ -119,7 +132,9 @@ public void testSearch() throws Exception { mappingsAndSettings.endObject(); } mappingsAndSettings.endObject(); - mappingsAndSettings.endObject(); + if (isRunningAgainstAncientCluster()) { + mappingsAndSettings.endObject(); + } mappingsAndSettings.endObject(); } mappingsAndSettings.endObject(); @@ -132,17 +147,20 @@ public void testSearch() throws Exception { count = randomIntBetween(2000, 3000); byte[] randomByteArray = new byte[16]; random().nextBytes(randomByteArray); - indexRandomDocuments(count, true, true, i -> { - return JsonXContent.contentBuilder().startObject() - .field("string", randomAlphaOfLength(10)) - .field("int", randomInt(100)) - .field("float", randomFloat()) - // be sure to create a "proper" boolean (True, False) for the first document so that automapping is correct - .field("bool", i > 0 && supportsLenientBooleans ? randomLenientBoolean() : randomBoolean()) - .field("field.with.dots", randomAlphaOfLength(10)) - .field("binary", Base64.getEncoder().encodeToString(randomByteArray)) - .endObject(); - }); + indexRandomDocuments( + count, + true, + true, + i -> JsonXContent.contentBuilder().startObject() + .field("string", randomAlphaOfLength(10)) + .field("int", randomInt(100)) + .field("float", randomFloat()) + // be sure to create a "proper" boolean (True, False) for the first document so that automapping is correct + .field("bool", i > 0 && supportsLenientBooleans ? randomLenientBoolean() : randomBoolean()) + .field("field.with.dots", randomAlphaOfLength(10)) + .field("binary", Base64.getEncoder().encodeToString(randomByteArray)) + .endObject() + ); refresh(); } else { count = countOfIndexedRandomDocuments(); @@ -152,7 +170,7 @@ public void testSearch() throws Exception { assertBasicSearchWorks(count); assertAllSearchWorks(count); assertBasicAggregationWorks(); - assertRealtimeGetWorks(); + assertRealtimeGetWorks(type); assertStoredBinaryFields(count); } @@ -168,7 +186,9 @@ public void testNewReplicasWork() throws Exception { } { mappingsAndSettings.startObject("mappings"); - mappingsAndSettings.startObject("doc"); + if (isRunningAgainstAncientCluster()) { + mappingsAndSettings.startObject(type); + } mappingsAndSettings.startObject("properties"); { mappingsAndSettings.startObject("field"); @@ -176,7 +196,9 @@ public void testNewReplicasWork() throws Exception { mappingsAndSettings.endObject(); } mappingsAndSettings.endObject(); - mappingsAndSettings.endObject(); + if (isRunningAgainstAncientCluster()) { + mappingsAndSettings.endObject(); + } mappingsAndSettings.endObject(); } mappingsAndSettings.endObject(); @@ -187,11 +209,8 @@ public void testNewReplicasWork() throws Exception { client().performRequest(createIndex); int numDocs = randomIntBetween(2000, 3000); - indexRandomDocuments(numDocs, true, false, i -> { - return JsonXContent.contentBuilder().startObject() - .field("field", "value") - .endObject(); - }); + indexRandomDocuments( + numDocs, true, false, i -> JsonXContent.contentBuilder().startObject().field("field", "value").endObject()); logger.info("Refreshing [{}]", index); client().performRequest(new Request("POST", "/" + index + "/_refresh")); } else { @@ -200,7 +219,7 @@ public void testNewReplicasWork() throws Exception { logger.debug("--> creating [{}] replicas for index [{}]", numReplicas, index); Request setNumberOfReplicas = new Request("PUT", "/" + index + "/_settings"); setNumberOfReplicas.setJsonEntity("{ \"index\": { \"number_of_replicas\" : " + numReplicas + " }}"); - Response response = client().performRequest(setNumberOfReplicas); + client().performRequest(setNumberOfReplicas); ensureGreenLongWait(index); @@ -221,76 +240,6 @@ public void testNewReplicasWork() throws Exception { } } - /** - * Search on an alias that contains illegal characters that would prevent it from being created after 5.1.0. It should still be - * search-able though. - */ - public void testAliasWithBadName() throws Exception { - assumeTrue("Can only test bad alias name if old cluster is on 5.1.0 or before", - getOldClusterVersion().before(VERSION_5_1_0_UNRELEASED)); - - int count; - if (isRunningAgainstOldCluster()) { - XContentBuilder mappingsAndSettings = jsonBuilder(); - mappingsAndSettings.startObject(); - { - mappingsAndSettings.startObject("settings"); - mappingsAndSettings.field("number_of_shards", 1); - mappingsAndSettings.field("number_of_replicas", 0); - mappingsAndSettings.endObject(); - } - { - mappingsAndSettings.startObject("mappings"); - mappingsAndSettings.startObject("doc"); - mappingsAndSettings.startObject("properties"); - { - mappingsAndSettings.startObject("key"); - mappingsAndSettings.field("type", "keyword"); - mappingsAndSettings.endObject(); - } - mappingsAndSettings.endObject(); - mappingsAndSettings.endObject(); - mappingsAndSettings.endObject(); - } - mappingsAndSettings.endObject(); - Request createIndex = new Request("PUT", "/" + index); - createIndex.setJsonEntity(Strings.toString(mappingsAndSettings)); - client().performRequest(createIndex); - - String aliasName = "%23" + index; // %23 == # - client().performRequest(new Request("PUT", "/" + index + "/_alias/" + aliasName)); - Response response = client().performRequest(new Request("HEAD", "/" + index + "/_alias/" + aliasName)); - assertEquals(200, response.getStatusLine().getStatusCode()); - - count = randomIntBetween(32, 128); - indexRandomDocuments(count, true, true, i -> { - return JsonXContent.contentBuilder().startObject() - .field("key", "value") - .endObject(); - }); - refresh(); - } else { - count = countOfIndexedRandomDocuments(); - } - - Request request = new Request("GET", "/_cluster/state"); - request.addParameter("metric", "metadata"); - logger.error("clusterState=" + entityAsMap(client().performRequest(request))); - // We can read from the alias just like we can read from the index. - String aliasName = "%23" + index; // %23 == # - Map searchRsp = entityAsMap(client().performRequest(new Request("GET", "/" + aliasName + "/_search"))); - int totalHits = extractTotalHits(searchRsp); - assertEquals(count, totalHits); - if (isRunningAgainstOldCluster() == false) { - // We can remove the alias. - Response response = client().performRequest(new Request("DELETE", "/" + index + "/_alias/" + aliasName)); - assertEquals(200, response.getStatusLine().getStatusCode()); - // and check that it is gone: - response = client().performRequest(new Request("HEAD", "/" + index + "/_alias/" + aliasName)); - assertEquals(404, response.getStatusLine().getStatusCode()); - } - } - public void testClusterState() throws Exception { if (isRunningAgainstOldCluster()) { XContentBuilder mappingsAndSettings = jsonBuilder(); @@ -344,16 +293,32 @@ public void testShrink() throws IOException { mappingsAndSettings.startObject(); { mappingsAndSettings.startObject("mappings"); - mappingsAndSettings.startObject("doc"); - mappingsAndSettings.startObject("properties"); { - mappingsAndSettings.startObject("field"); - mappingsAndSettings.field("type", "text"); + if (isRunningAgainstAncientCluster()) { + mappingsAndSettings.startObject(type); + } + mappingsAndSettings.startObject("properties"); + { + mappingsAndSettings.startObject("field"); + { + mappingsAndSettings.field("type", "text"); + } + mappingsAndSettings.endObject(); + } mappingsAndSettings.endObject(); + if (isRunningAgainstAncientCluster()) { + mappingsAndSettings.endObject(); + } } mappingsAndSettings.endObject(); - mappingsAndSettings.endObject(); - mappingsAndSettings.endObject(); + if (isRunningAgainstAncientCluster() == false) { + // the default number of shards is now one so we have to set the number of shards to be more than one explicitly + mappingsAndSettings.startObject("settings"); + { + mappingsAndSettings.field("index.number_of_shards", 5); + } + mappingsAndSettings.endObject(); + } } mappingsAndSettings.endObject(); @@ -363,11 +328,8 @@ public void testShrink() throws IOException { client().performRequest(createIndex); numDocs = randomIntBetween(512, 1024); - indexRandomDocuments(numDocs, true, true, i -> { - return JsonXContent.contentBuilder().startObject() - .field("field", "value") - .endObject(); - }); + indexRandomDocuments( + numDocs, true, true, i -> JsonXContent.contentBuilder().startObject().field("field", "value").endObject()); ensureGreen(index); // wait for source index to be available on both nodes before starting shrink @@ -376,7 +338,7 @@ public void testShrink() throws IOException { client().performRequest(updateSettingsRequest); Request shrinkIndexRequest = new Request("PUT", "/" + index + "/_shrink/" + shrunkenIndex); - if (getOldClusterVersion().onOrAfter(Version.V_6_4_0)) { + if (getOldClusterVersion().onOrAfter(Version.V_6_4_0) && getOldClusterVersion().before(Version.V_7_0_0)) { shrinkIndexRequest.addParameter("copy_settings", "true"); } shrinkIndexRequest.setJsonEntity("{\"settings\": {\"index.number_of_shards\": 1}}"); @@ -414,16 +376,30 @@ public void testShrinkAfterUpgrade() throws IOException { mappingsAndSettings.startObject(); { mappingsAndSettings.startObject("mappings"); - mappingsAndSettings.startObject("doc"); - mappingsAndSettings.startObject("properties"); { - mappingsAndSettings.startObject("field"); - mappingsAndSettings.field("type", "text"); + if (isRunningAgainstAncientCluster()) { + mappingsAndSettings.startObject(type); + } + mappingsAndSettings.startObject("properties"); + { + mappingsAndSettings.startObject("field"); + { + mappingsAndSettings.field("type", "text"); + } + mappingsAndSettings.endObject(); + } mappingsAndSettings.endObject(); + if (isRunningAgainstAncientCluster()) { + mappingsAndSettings.endObject(); + } } mappingsAndSettings.endObject(); - mappingsAndSettings.endObject(); - mappingsAndSettings.endObject(); + if (isRunningAgainstAncientCluster() == false) { + // the default number of shards is now one so we have to set the number of shards to be more than one explicitly + mappingsAndSettings.startObject("settings"); + mappingsAndSettings.field("index.number_of_shards", 5); + mappingsAndSettings.endObject(); + } } mappingsAndSettings.endObject(); @@ -433,11 +409,12 @@ public void testShrinkAfterUpgrade() throws IOException { client().performRequest(createIndex); numDocs = randomIntBetween(512, 1024); - indexRandomDocuments(numDocs, true, true, i -> { - return JsonXContent.contentBuilder().startObject() - .field("field", "value") - .endObject(); - }); + indexRandomDocuments( + numDocs, + true, + true, + i -> JsonXContent.contentBuilder().startObject().field("field", "value").endObject() + ); } else { ensureGreen(index); // wait for source index to be available on both nodes before starting shrink @@ -504,7 +481,7 @@ public void testRollover() throws IOException { bulk.append("{\"index\":{}}\n"); bulk.append("{\"test\":\"test\"}\n"); } - Request bulkRequest = new Request("POST", "/" + index + "_write/doc/_bulk"); + Request bulkRequest = new Request("POST", "/" + index + "_write/" + type + "/_bulk"); bulkRequest.setJsonEntity(bulk.toString()); bulkRequest.addParameter("refresh", ""); bulkRequest.setOptions(expectWarnings(RestBulkAction.TYPES_DEPRECATION_MESSAGE)); @@ -627,7 +604,7 @@ void assertBasicAggregationWorks() throws IOException { assertTotalHits(termsCount, boolTerms); } - void assertRealtimeGetWorks() throws IOException { + void assertRealtimeGetWorks(final String typeName) throws IOException { Request disableAutoRefresh = new Request("PUT", "/" + index + "/_settings"); disableAutoRefresh.setJsonEntity("{ \"index\": { \"refresh_interval\" : -1 }}"); client().performRequest(disableAutoRefresh); @@ -638,13 +615,15 @@ void assertRealtimeGetWorks() throws IOException { Map hit = (Map) ((List)(XContentMapValues.extractValue("hits.hits", searchResponse))).get(0); String docId = (String) hit.get("_id"); - Request updateRequest = new Request("POST", "/" + index + "/doc/" + docId + "/_update"); + Request updateRequest = new Request("POST", "/" + index + "/" + typeName + "/" + docId + "/_update"); updateRequest.setOptions(expectWarnings(RestUpdateAction.TYPES_DEPRECATION_MESSAGE)); updateRequest.setJsonEntity("{ \"doc\" : { \"foo\": \"bar\"}}"); client().performRequest(updateRequest); - Request getRequest = new Request("GET", "/" + index + "/doc/" + docId); - getRequest.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE)); + Request getRequest = new Request("GET", "/" + index + "/" + typeName + "/" + docId); + if (getOldClusterVersion().before(Version.V_6_7_0)) { + getRequest.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE)); + } Map getRsp = entityAsMap(client().performRequest(getRequest)); Map source = (Map) getRsp.get("_source"); assertTrue("doc does not contain 'foo' key: " + source, source.containsKey("foo")); @@ -683,7 +662,7 @@ static void assertNoFailures(Map response) { void assertTotalHits(int expectedTotalHits, Map response) { int actualTotalHits = extractTotalHits(response); - assertEquals(expectedTotalHits, actualTotalHits); + assertEquals(response.toString(), expectedTotalHits, actualTotalHits); } int extractTotalHits(Map response) { @@ -698,7 +677,7 @@ int extractTotalHits(Map response) { * Tests that a single document survives. Super basic smoke test. */ public void testSingleDoc() throws IOException { - String docLocation = "/" + index + "/doc/1"; + String docLocation = "/" + index + "/" + type + "/1"; String doc = "{\"test\": \"test\"}"; if (isRunningAgainstOldCluster()) { @@ -709,7 +688,9 @@ public void testSingleDoc() throws IOException { Request request = new Request("GET", docLocation); - request.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE)); + if (getOldClusterVersion().before(Version.V_6_7_0)) { + request.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE)); + } assertThat(toStr(client().performRequest(request)), containsString(doc)); } @@ -773,8 +754,12 @@ public void testRecovery() throws Exception { } if (shouldHaveTranslog) { // Update a few documents so we are sure to have a translog - indexRandomDocuments(count / 10, false /* Flushing here would invalidate the whole thing....*/, false, - i -> jsonBuilder().startObject().field("field", "value").endObject()); + indexRandomDocuments( + count / 10, + false, // flushing here would invalidate the whole thing + false, + i -> jsonBuilder().startObject().field("field", "value").endObject() + ); } saveInfoDocument("should_have_translog", Boolean.toString(shouldHaveTranslog)); } else { @@ -785,6 +770,7 @@ public void testRecovery() throws Exception { // Count the documents in the index to make sure we have as many as we put there Request countRequest = new Request("GET", "/" + index + "/_search"); countRequest.addParameter("size", "0"); + refresh(); Map countResponse = entityAsMap(client().performRequest(countRequest)); assertTotalHits(count, countResponse); @@ -888,13 +874,19 @@ public void testSnapshotRestore() throws IOException { } templateBuilder.endObject(); templateBuilder.startObject("mappings"); { - templateBuilder.startObject("doc"); { - templateBuilder.startObject("_source"); { + if (isRunningAgainstAncientCluster()) { + templateBuilder.startObject(type); + } + { + templateBuilder.startObject("_source"); + { templateBuilder.field("enabled", true); } templateBuilder.endObject(); } - templateBuilder.endObject(); + if (isRunningAgainstAncientCluster()) { + templateBuilder.endObject(); + } } templateBuilder.endObject(); templateBuilder.startObject("aliases"); { @@ -913,12 +905,6 @@ public void testSnapshotRestore() throws IOException { templateBuilder.endObject().endObject(); Request createTemplateRequest = new Request("PUT", "/_template/test_template"); createTemplateRequest.setJsonEntity(Strings.toString(templateBuilder)); - - // In 7.0, type names are no longer expected by default in put index template requests. - // We therefore use the deprecated typed APIs when running against the current version. - if (isRunningAgainstOldCluster() == false) { - createTemplateRequest.addParameter(INCLUDE_TYPE_NAME_PARAMETER, "true"); - } createTemplateRequest.setOptions(allowTypesRemovalWarnings()); client().performRequest(createTemplateRequest); @@ -1010,12 +996,13 @@ public void testSoftDeletes() throws Exception { int numDocs = between(10, 100); for (int i = 0; i < numDocs; i++) { String doc = Strings.toString(JsonXContent.contentBuilder().startObject().field("field", "v1").endObject()); - Request request = new Request("POST", "/" + index + "/doc/" + i); + Request request = new Request("POST", "/" + index + "/" + type + "/" + i); + if (isRunningAgainstAncientCluster() == false) { + request.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE)); + } request.setJsonEntity(doc); client().performRequest(request); - if (rarely()) { - refresh(); - } + refresh(); } client().performRequest(new Request("POST", "/" + index + "/_flush")); int liveDocs = numDocs; @@ -1023,11 +1010,11 @@ public void testSoftDeletes() throws Exception { for (int i = 0; i < numDocs; i++) { if (randomBoolean()) { String doc = Strings.toString(JsonXContent.contentBuilder().startObject().field("field", "v2").endObject()); - Request request = new Request("POST", "/" + index + "/doc/" + i); + Request request = new Request("POST", "/" + index + "/" + type + "/" + i); request.setJsonEntity(doc); client().performRequest(request); } else if (randomBoolean()) { - client().performRequest(new Request("DELETE", "/" + index + "/doc/" + i)); + client().performRequest(new Request("DELETE", "/" + index + "/" + type + "/" + i)); liveDocs--; } } @@ -1040,7 +1027,7 @@ public void testSoftDeletes() throws Exception { } } - private void checkSnapshot(String snapshotName, int count, Version tookOnVersion) throws IOException { + private void checkSnapshot(final String snapshotName, final int count, final Version tookOnVersion) throws IOException { // Check the snapshot metadata, especially the version Request listSnapshotRequest = new Request("GET", "/_snapshot/repo/" + snapshotName); Map listSnapshotResponse = entityAsMap(client().performRequest(listSnapshotRequest)); @@ -1097,7 +1084,7 @@ && getOldClusterVersion().onOrAfter(Version.V_6_1_0) && getOldClusterVersion().b bulk.append("{\"index\":{\"_id\":\"").append(count + i).append("\"}}\n"); bulk.append("{\"test\":\"test\"}\n"); } - Request writeToRestoredRequest = new Request("POST", "/restored_" + index + "/doc/_bulk"); + Request writeToRestoredRequest = new Request("POST", "/restored_" + index + "/" + type + "/_bulk"); writeToRestoredRequest.addParameter("refresh", "true"); writeToRestoredRequest.setJsonEntity(bulk.toString()); writeToRestoredRequest.setOptions(expectWarnings(RestBulkAction.TYPES_DEPRECATION_MESSAGE)); @@ -1123,12 +1110,6 @@ && getOldClusterVersion().onOrAfter(Version.V_6_1_0) && getOldClusterVersion().b // Check that the template was restored successfully Request getTemplateRequest = new Request("GET", "/_template/test_template"); - - // In 7.0, type names are no longer returned by default in get index template requests. - // We therefore use the deprecated typed APIs when running against the current version. - if (isRunningAgainstOldCluster() == false) { - getTemplateRequest.addParameter(INCLUDE_TYPE_NAME_PARAMETER, "true"); - } getTemplateRequest.setOptions(allowTypesRemovalWarnings()); Map getTemplateResponse = entityAsMap(client().performRequest(getTemplateRequest)); @@ -1139,7 +1120,14 @@ && getOldClusterVersion().onOrAfter(Version.V_6_1_0) && getOldClusterVersion().b expectedTemplate.put("index_patterns", singletonList("evil_*")); } expectedTemplate.put("settings", singletonMap("index", singletonMap("number_of_shards", "1"))); - expectedTemplate.put("mappings", singletonMap("doc", singletonMap("_source", singletonMap("enabled", true)))); + // We don't have the type in the response starting with 7.0, but we won't have it on old cluster after upgrade + // either so look at the response to figure out the correct assertions + if (isTypeInTemplateResponse(getTemplateResponse)) { + expectedTemplate.put("mappings", singletonMap(type, singletonMap("_source", singletonMap("enabled", true)))); + } else { + expectedTemplate.put("mappings", singletonMap("_source", singletonMap("enabled", true))); + } + expectedTemplate.put("order", 0); Map aliases = new HashMap<>(); aliases.put("alias1", emptyMap()); @@ -1149,18 +1137,33 @@ && getOldClusterVersion().onOrAfter(Version.V_6_1_0) && getOldClusterVersion().b if (false == expectedTemplate.equals(getTemplateResponse)) { NotEqualMessageBuilder builder = new NotEqualMessageBuilder(); builder.compareMaps(getTemplateResponse, expectedTemplate); + logger.info("expected: {}\nactual:{}", expectedTemplate, getTemplateResponse); fail("template doesn't match:\n" + builder.toString()); } } + @SuppressWarnings("unchecked") + private boolean isTypeInTemplateResponse(Map getTemplateResponse) { + return ( (Map) ( + (Map) getTemplateResponse.getOrDefault("test_template", emptyMap()) + ).get("mappings")).get("_source") == null; + } + // TODO tests for upgrades after shrink. We've had trouble with shrink in the past. - private void indexRandomDocuments(int count, boolean flushAllowed, boolean saveInfo, - CheckedFunction docSupplier) throws IOException { + private void indexRandomDocuments( + final int count, + final boolean flushAllowed, + final boolean saveInfo, + final CheckedFunction docSupplier) + throws IOException { logger.info("Indexing {} random documents", count); for (int i = 0; i < count; i++) { logger.debug("Indexing document [{}]", i); - Request createDocument = new Request("POST", "/" + index + "/doc/" + i); + Request createDocument = new Request("POST", "/" + index + "/" + type + "/" + i); + if (isRunningAgainstAncientCluster() == false) { + createDocument.setOptions(expectWarnings(RestBulkAction.TYPES_DEPRECATION_MESSAGE)); + } createDocument.setJsonEntity(Strings.toString(docSupplier.apply(i))); client().performRequest(createDocument); if (rarely()) { @@ -1185,16 +1188,21 @@ private void saveInfoDocument(String type, String value) throws IOException { infoDoc.field("value", value); infoDoc.endObject(); // Only create the first version so we know how many documents are created when the index is first created - Request request = new Request("PUT", "/info/doc/" + index + "_" + type); + Request request = new Request("PUT", "/info/" + this.type + "/" + index + "_" + type); request.addParameter("op_type", "create"); request.setJsonEntity(Strings.toString(infoDoc)); + if (isRunningAgainstAncientCluster() == false) { + request.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE)); + } client().performRequest(request); } private String loadInfoDocument(String type) throws IOException { - Request request = new Request("GET", "/info/doc/" + index + "_" + type); + Request request = new Request("GET", "/info/" + this.type + "/" + index + "_" + type); request.addParameter("filter_path", "_source"); - request.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE)); + if (getOldClusterVersion().before(Version.V_6_7_0)) { + request.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE)); + } String doc = toStr(client().performRequest(request)); Matcher m = Pattern.compile("\"value\":\"(.+)\"").matcher(doc); assertTrue(doc, m.find()); diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java index 3030663c35d2d..fb4e33863cacf 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java @@ -20,6 +20,7 @@ package org.elasticsearch.upgrades; import org.apache.http.util.EntityUtils; +import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.common.Strings; @@ -143,6 +144,7 @@ private static void addCandidate(String querySource, QueryBuilder expectedQb) { } public void testQueryBuilderBWC() throws Exception { + final String type = getOldClusterVersion().before(Version.V_7_0_0) ? "doc" : "_doc"; String index = "queries"; if (isRunningAgainstOldCluster()) { XContentBuilder mappingsAndSettings = jsonBuilder(); @@ -155,7 +157,9 @@ public void testQueryBuilderBWC() throws Exception { } { mappingsAndSettings.startObject("mappings"); - mappingsAndSettings.startObject("doc"); + if (isRunningAgainstAncientCluster()) { + mappingsAndSettings.startObject(type); + } mappingsAndSettings.startObject("properties"); { mappingsAndSettings.startObject("query"); @@ -174,7 +178,9 @@ public void testQueryBuilderBWC() throws Exception { } mappingsAndSettings.endObject(); mappingsAndSettings.endObject(); - mappingsAndSettings.endObject(); + if (isRunningAgainstAncientCluster()) { + mappingsAndSettings.endObject(); + } } mappingsAndSettings.endObject(); Request request = new Request("PUT", "/" + index); @@ -184,7 +190,7 @@ public void testQueryBuilderBWC() throws Exception { assertEquals(200, rsp.getStatusLine().getStatusCode()); for (int i = 0; i < CANDIDATES.size(); i++) { - request = new Request("PUT", "/" + index + "/doc/" + Integer.toString(i)); + request = new Request("PUT", "/" + index + "/" + type + "/" + Integer.toString(i)); request.setJsonEntity((String) CANDIDATES.get(i)[0]); rsp = client().performRequest(request); assertEquals(201, rsp.getStatusLine().getStatusCode()); diff --git a/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java b/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java index 508949b561fb4..6d1d6839e1cc5 100644 --- a/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java @@ -63,6 +63,14 @@ public final boolean isRunningAgainstOldCluster() { private final Version oldClusterVersion = Version.fromString(System.getProperty("tests.old_cluster_version")); + /** + * @return true if test is running against an old cluster before that last major, in this case + * when System.getProperty("tests.is_old_cluster" == true) and oldClusterVersion is before {@link Version#V_7_0_0} + */ + protected final boolean isRunningAgainstAncientCluster() { + return isRunningAgainstOldCluster() && oldClusterVersion.before(Version.V_7_0_0); + } + public final Version getOldClusterVersion() { return oldClusterVersion; } From 4bf1cbcb14bbfc19da51b4337f9ea91eb97149d8 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Tue, 16 Apr 2019 14:57:59 -0500 Subject: [PATCH 2/3] fix checkStyle --- .../org/elasticsearch/upgrades/FullClusterRestartIT.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index a4174d661ed1d..97f87d10230a1 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -27,7 +27,6 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.client.WarningFailureException; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.MetaDataIndexStateService; import org.elasticsearch.common.Booleans; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.Strings; @@ -40,7 +39,6 @@ import org.elasticsearch.rest.action.document.RestIndexAction; import org.elasticsearch.rest.action.document.RestUpdateAction; import org.elasticsearch.rest.action.search.RestExplainAction; - import org.elasticsearch.test.NotEqualMessageBuilder; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.yaml.ObjectPath; @@ -49,7 +47,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Base64; -import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -68,11 +65,8 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.startsWith; /** From 16da99e5b6b35c90bae98fc5969ca9f2f3691872 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Tue, 16 Apr 2019 17:48:53 -0500 Subject: [PATCH 3/3] add xpack tests --- .../xpack/restart/FullClusterRestartIT.java | 207 ++++++++++++------ 1 file changed, 136 insertions(+), 71 deletions(-) diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java index b89e78b91aecf..322e97db765ff 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java @@ -13,7 +13,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.ObjectPath; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.action.document.RestGetAction; @@ -22,17 +21,8 @@ import org.elasticsearch.test.StreamsUtils; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.upgrades.AbstractFullClusterRestartTestCase; -import org.elasticsearch.xpack.core.upgrade.UpgradeField; -import org.elasticsearch.xpack.core.watcher.client.WatchSourceBuilder; -import org.elasticsearch.xpack.security.support.SecurityIndexManager; -import org.elasticsearch.xpack.watcher.actions.index.IndexAction; -import org.elasticsearch.xpack.watcher.actions.logging.LoggingAction; -import org.elasticsearch.xpack.watcher.common.text.TextTemplate; -import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition; -import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest; -import org.elasticsearch.xpack.watcher.trigger.schedule.IntervalSchedule; -import org.elasticsearch.xpack.watcher.trigger.schedule.ScheduleTrigger; import org.hamcrest.Matcher; +import org.junit.Before; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -60,6 +50,22 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase { + public static final String INDEX_ACTION_TYPES_DEPRECATION_MESSAGE = + "[types removal] Specifying types in a watcher index action is deprecated."; + + public static final String SEARCH_INPUT_TYPES_DEPRECATION_MESSAGE = + "[types removal] Specifying types in a watcher search request is deprecated."; + + public static final int UPGRADE_FIELD_EXPECTED_INDEX_FORMAT_VERSION = 6; + public static final int SECURITY_EXPECTED_INDEX_FORMAT_VERSION = 6; + + private String type; + + @Before + public void setType() { + type = getOldClusterVersion().before(Version.V_6_7_0) ? "doc" : "_doc"; + } + @Override protected Settings restClientSettings() { String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8)); @@ -76,7 +82,7 @@ protected Settings restClientSettings() { * Tests that a single document survives. Super basic smoke test. */ public void testSingleDoc() throws IOException { - String docLocation = "/testsingledoc/doc/1"; + String docLocation = "/testsingledoc/" + type + "/1"; String doc = "{\"test\": \"test\"}"; if (isRunningAgainstOldCluster()) { @@ -87,7 +93,9 @@ public void testSingleDoc() throws IOException { } Request getRequest = new Request("GET", docLocation); - getRequest.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE)); + if (getOldClusterVersion().before(Version.V_6_7_0)) { + getRequest.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE)); + } assertThat(toStr(client().performRequest(getRequest)), containsString(doc)); } @@ -113,7 +121,7 @@ public void testSecurityNativeRealm() throws Exception { if (settingsMap.containsKey("index")) { @SuppressWarnings("unchecked") int format = Integer.parseInt(String.valueOf(((Map)settingsMap.get("index")).get("format"))); - assertEquals("The security index needs to be upgraded", SecurityIndexManager.INTERNAL_INDEX_FORMAT, format); + assertEquals("The security index needs to be upgraded", SECURITY_EXPECTED_INDEX_FORMAT_VERSION, format); } } @@ -131,17 +139,21 @@ public void testSecurityNativeRealm() throws Exception { public void testWatcher() throws Exception { if (isRunningAgainstOldCluster()) { logger.info("Adding a watch on old cluster {}", getOldClusterVersion()); - Request createBwcWatch = new Request("PUT", "/_xpack/watcher/watch/bwc_watch"); + Request createBwcWatch = new Request("PUT", getWatcherEndpoint() + "/watch/bwc_watch"); + Request createBwcThrottlePeriod = new Request("PUT", getWatcherEndpoint() + "/watch/bwc_throttle_period"); + if (getOldClusterVersion().onOrAfter(Version.V_7_0_0)) { + createBwcWatch.setOptions(expectWarnings(INDEX_ACTION_TYPES_DEPRECATION_MESSAGE)); + createBwcThrottlePeriod.setOptions(expectWarnings(INDEX_ACTION_TYPES_DEPRECATION_MESSAGE)); + } createBwcWatch.setJsonEntity(loadWatch("simple-watch.json")); client().performRequest(createBwcWatch); logger.info("Adding a watch with \"fun\" throttle periods on old cluster"); - Request createBwcThrottlePeriod = new Request("PUT", "_xpack/watcher/watch/bwc_throttle_period"); createBwcThrottlePeriod.setJsonEntity(loadWatch("throttle-period-watch.json")); client().performRequest(createBwcThrottlePeriod); logger.info("Adding a watch with \"fun\" read timeout on old cluster"); - Request createFunnyTimeout = new Request("PUT", "_xpack/watcher/watch/bwc_funny_timeout"); + Request createFunnyTimeout = new Request("PUT", getWatcherEndpoint() + "/watch/bwc_funny_timeout"); createFunnyTimeout.setJsonEntity(loadWatch("funny-timeout-watch.json")); client().performRequest(createFunnyTimeout); @@ -169,7 +181,7 @@ public void testWatcher() throws Exception { logger.info("settings map {}", settingsMap); if (settingsMap.containsKey("index")) { int format = Integer.parseInt(String.valueOf(((Map)settingsMap.get("index")).get("format"))); - assertEquals("The watches index needs to be upgraded", UpgradeField.EXPECTED_INDEX_FORMAT_VERSION, format); + assertEquals("The watches index needs to be upgraded", UPGRADE_FIELD_EXPECTED_INDEX_FORMAT_VERSION, format); } } @@ -218,7 +230,11 @@ public void testRollupAfterRestart() throws Exception { // index documents for the rollup job final StringBuilder bulk = new StringBuilder(); for (int i = 0; i < numDocs; i++) { - bulk.append("{\"index\":{\"_index\":\"rollup-docs\",\"_type\":\"doc\"}}\n"); + if (getOldClusterVersion().onOrAfter(Version.V_7_0_0)) { + bulk.append("{\"index\":{\"_index\":\"rollup-docs\"}}\n"); + } else { + bulk.append("{\"index\":{\"_index\":\"rollup-docs\",\"_type\":\"doc\"}}\n"); + } String date = String.format(Locale.ROOT, "%04d-01-01T00:%02d:00Z", year, i); bulk.append("{\"timestamp\":\"").append(date).append("\",\"value\":").append(i).append("}\n"); } @@ -229,7 +245,8 @@ public void testRollupAfterRestart() throws Exception { client().performRequest(bulkRequest); // create the rollup job - final Request createRollupJobRequest = new Request("PUT", "/_xpack/rollup/job/rollup-job-test"); + final Request createRollupJobRequest = new Request("PUT", getRollupEndpoint() + "/job/rollup-job-test"); + createRollupJobRequest.setJsonEntity("{" + "\"index_pattern\":\"rollup-*\"," + "\"rollup_index\":\"results-rollup\"," @@ -250,7 +267,7 @@ public void testRollupAfterRestart() throws Exception { assertThat(createRollupJobResponse.get("acknowledged"), equalTo(Boolean.TRUE)); // start the rollup job - final Request startRollupJobRequest = new Request("POST", "/_xpack/rollup/job/rollup-job-test/_start"); + final Request startRollupJobRequest = new Request("POST", getRollupEndpoint() + "/job/rollup-job-test/_start"); Map startRollupJobResponse = entityAsMap(client().performRequest(startRollupJobRequest)); assertThat(startRollupJobResponse.get("started"), equalTo(Boolean.TRUE)); @@ -276,12 +293,12 @@ public void testRollupIDSchemeAfterRestart() throws Exception { assumeTrue("Rollup ID scheme changed in 6.4", getOldClusterVersion().before(Version.V_6_4_0)); if (isRunningAgainstOldCluster()) { - final Request indexRequest = new Request("POST", "/id-test-rollup/doc/1"); + final Request indexRequest = new Request("POST", "/id-test-rollup/" + type + "/1"); indexRequest.setJsonEntity("{\"timestamp\":\"2018-01-01T00:00:01\",\"value\":123}"); client().performRequest(indexRequest); // create the rollup job - final Request createRollupJobRequest = new Request("PUT", "/_xpack/rollup/job/rollup-id-test"); + final Request createRollupJobRequest = new Request("PUT", getRollupEndpoint() + "/job/rollup-id-test"); createRollupJobRequest.setJsonEntity("{" + "\"index_pattern\":\"id-test-rollup\"," + "\"rollup_index\":\"id-test-results-rollup\"," @@ -309,7 +326,7 @@ public void testRollupIDSchemeAfterRestart() throws Exception { assertThat(createRollupJobResponse.get("acknowledged"), equalTo(Boolean.TRUE)); // start the rollup job - final Request startRollupJobRequest = new Request("POST", "/_xpack/rollup/job/rollup-id-test/_start"); + final Request startRollupJobRequest = new Request("POST", getRollupEndpoint() + "/job/rollup-id-test/_start"); Map startRollupJobResponse = entityAsMap(client().performRequest(startRollupJobRequest)); assertThat(startRollupJobResponse.get("started"), equalTo(Boolean.TRUE)); @@ -337,9 +354,11 @@ public void testRollupIDSchemeAfterRestart() throws Exception { } else { - final Request indexRequest = new Request("POST", "/id-test-rollup/doc/2"); + final Request indexRequest = new Request("POST", "/id-test-rollup/" + type + "/2"); indexRequest.setJsonEntity("{\"timestamp\":\"2018-01-02T00:00:01\",\"value\":345}"); - indexRequest.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE)); + if (getOldClusterVersion().before(Version.V_6_7_0)) { + indexRequest.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE)); + } client().performRequest(indexRequest); assertRollUpJob("rollup-id-test"); @@ -403,12 +422,8 @@ public void testSqlFailsOnIndexWithTwoTypes() throws IOException { client().performRequest(doc2); return; } - final Request sqlRequest; - if (isRunningAgainstOldCluster()) { - sqlRequest = new Request("POST", "/_xpack/sql"); - } else { - sqlRequest = new Request("POST", "/_sql"); - } + final Request sqlRequest = new Request("POST", getSQLEndpoint()); + sqlRequest.setJsonEntity("{\"query\":\"SELECT * FROM testsqlfailsonindexwithtwotypes\"}"); ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(sqlRequest)); assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); @@ -430,8 +445,21 @@ private void assertOldTemplatesAreDeleted() throws IOException { private void assertWatchIndexContentsWork() throws Exception { // Fetch a basic watch Request getRequest = new Request("GET", "_watcher/watch/bwc_watch"); - getRequest.setOptions(expectWarnings(IndexAction.TYPES_DEPRECATION_MESSAGE, - WatcherSearchTemplateRequest.TYPES_DEPRECATION_MESSAGE)); + if (getOldClusterVersion().before(Version.V_7_0_0)) { + getRequest.setOptions( + expectWarnings( + INDEX_ACTION_TYPES_DEPRECATION_MESSAGE, + SEARCH_INPUT_TYPES_DEPRECATION_MESSAGE + ) + ); + } else { + getRequest.setOptions( + expectWarnings( + INDEX_ACTION_TYPES_DEPRECATION_MESSAGE + ) + ); + } + Map bwcWatch = entityAsMap(client().performRequest(getRequest)); logger.error("-----> {}", bwcWatch); @@ -447,8 +475,20 @@ private void assertWatchIndexContentsWork() throws Exception { // Fetch a watch with "fun" throttle periods getRequest = new Request("GET", "_watcher/watch/bwc_throttle_period"); - getRequest.setOptions(expectWarnings(IndexAction.TYPES_DEPRECATION_MESSAGE, - WatcherSearchTemplateRequest.TYPES_DEPRECATION_MESSAGE)); + if (getOldClusterVersion().before(Version.V_7_0_0)) { + getRequest.setOptions( + expectWarnings( + INDEX_ACTION_TYPES_DEPRECATION_MESSAGE, + SEARCH_INPUT_TYPES_DEPRECATION_MESSAGE + ) + ); + } else { + getRequest.setOptions( + expectWarnings( + INDEX_ACTION_TYPES_DEPRECATION_MESSAGE + ) + ); + } bwcWatch = entityAsMap(client().performRequest(getRequest)); assertThat(bwcWatch.get("found"), equalTo(true)); source = (Map) bwcWatch.get("watch"); @@ -487,10 +527,9 @@ private void assertWatchIndexContentsWork() throws Exception { private void assertBasicWatchInteractions() throws Exception { - String watch = new WatchSourceBuilder() - .condition(InternalAlwaysCondition.INSTANCE) - .trigger(ScheduleTrigger.builder(new IntervalSchedule(IntervalSchedule.Interval.seconds(1)))) - .addAction("awesome", LoggingAction.builder(new TextTemplate("test"))).buildAsBytes(XContentType.JSON).utf8ToString(); + String watch = "{\"trigger\":{\"schedule\":{\"interval\":\"1s\"}},\"input\":{\"none\":{}}," + + "\"condition\":{\"always\":{}}," + + "\"actions\":{\"awesome\":{\"logging\":{\"level\":\"info\",\"text\":\"test\"}}}}"; Request createWatchRequest = new Request("PUT", "_watcher/watch/new_watch"); createWatchRequest.setJsonEntity(watch); Map createWatch = entityAsMap(client().performRequest(createWatchRequest)); @@ -532,7 +571,13 @@ private void waitForHits(String indexName, int expectedHits) throws Exception { try { Map response = entityAsMap(client().performRequest(request)); Map hits = (Map) response.get("hits"); - int total = (int) hits.get("total"); + logger.info("Hits are: {}", hits); + int total; + if (getOldClusterVersion().onOrAfter(Version.V_7_0_0) || isRunningAgainstOldCluster() == false) { + total = (int) ((Map) hits.get("total")).get("value"); + } else { + total = (int) hits.get("total"); + } assertThat(total, greaterThanOrEqualTo(expectedHits)); } catch (IOException ioe) { if (ioe instanceof ResponseException) { @@ -552,12 +597,7 @@ static String toStr(Response response) throws IOException { private void createUser(final boolean oldCluster) throws Exception { final String id = oldCluster ? "preupgrade_user" : "postupgrade_user"; - Request request; - if (oldCluster) { - request = new Request("PUT", "/_xpack/security/user/" + id); - } else { - request = new Request("PUT", "/_security/user/" + id); - } + Request request = new Request("PUT", getSecurityEndpoint() + "/user/" + id); request.setJsonEntity( "{\n" + " \"password\" : \"j@rV1s\",\n" + @@ -571,12 +611,7 @@ private void createUser(final boolean oldCluster) throws Exception { private void createRole(final boolean oldCluster) throws Exception { final String id = oldCluster ? "preupgrade_role" : "postupgrade_role"; - Request request; - if (oldCluster) { - request = new Request("PUT", "/_xpack/security/role/" + id); - } else { - request = new Request("PUT", "/_security/role/" + id); - } + Request request = new Request("PUT", getSecurityEndpoint() + "/role/" + id); request.setJsonEntity( "{\n" + " \"run_as\": [ \"abc\" ],\n" + @@ -597,20 +632,59 @@ private void createRole(final boolean oldCluster) throws Exception { private void assertUserInfo(final boolean oldCluster) throws Exception { final String user = oldCluster ? "preupgrade_user" : "postupgrade_user"; - Map response = oldCluster ? - entityAsMap(client().performRequest(new Request("GET", "/_xpack/security/user/" + user))) : - entityAsMap(client().performRequest(new Request("GET", "/_security/user/" + user))); + Request request = new Request("GET", getSecurityEndpoint() + "/user/" + user);; + Map response = entityAsMap(client().performRequest(request)); @SuppressWarnings("unchecked") Map userInfo = (Map) response.get(user); assertEquals(user + "@example.com", userInfo.get("email")); assertNotNull(userInfo.get("full_name")); assertNotNull(userInfo.get("roles")); } + private String getSecurityEndpoint() { + String securityEndpoint; + if (getOldClusterVersion().onOrAfter(Version.V_7_0_0) || isRunningAgainstOldCluster() == false) { + securityEndpoint = "/_security"; + } else { + securityEndpoint = "/_xpack/security"; + } + return securityEndpoint; + } + + private String getSQLEndpoint() { + String securityEndpoint; + if (getOldClusterVersion().onOrAfter(Version.V_7_0_0) || isRunningAgainstOldCluster() == false) { + securityEndpoint = "/_sql"; + } else { + securityEndpoint = "/_xpack/sql"; + } + return securityEndpoint; + } + + private String getRollupEndpoint() { + String securityEndpoint; + if (getOldClusterVersion().onOrAfter(Version.V_7_0_0) || isRunningAgainstOldCluster() == false) { + securityEndpoint = "/_rollup"; + } else { + securityEndpoint = "/_xpack/rollup"; + } + return securityEndpoint; + } + + private String getWatcherEndpoint() { + String securityEndpoint; + if (getOldClusterVersion().onOrAfter(Version.V_7_0_0) || isRunningAgainstOldCluster() == false) { + securityEndpoint = "/_watcher"; + } else { + securityEndpoint = "/_xpack/watcher"; + } + return securityEndpoint; + } + private void assertRoleInfo(final boolean oldCluster) throws Exception { final String role = oldCluster ? "preupgrade_role" : "postupgrade_role"; - @SuppressWarnings("unchecked") Map response = oldCluster ? - (Map) entityAsMap(client().performRequest(new Request("GET", "/_xpack/security/role/" + role))).get(role) : - (Map) entityAsMap(client().performRequest(new Request("GET", "/_security/role/" + role))).get(role); + @SuppressWarnings("unchecked") Map response = (Map) entityAsMap( + client().performRequest(new Request("GET", getSecurityEndpoint() + "/role/" + role)) + ).get(role); assertNotNull(response.get("run_as")); assertNotNull(response.get("cluster")); assertNotNull(response.get("indices")); @@ -622,12 +696,7 @@ private void assertRollUpJob(final String rollupJob) throws Exception { waitForRollUpJob(rollupJob, expectedStates); // check that the rollup job is started using the RollUp API - final Request getRollupJobRequest; - if (isRunningAgainstOldCluster()) { - getRollupJobRequest = new Request("GET", "/_xpack/rollup/job/" + rollupJob); - } else { - getRollupJobRequest = new Request("GET", "/_rollup/job/" + rollupJob); - } + final Request getRollupJobRequest = new Request("GET", getRollupEndpoint() + "/job/" + rollupJob); Map getRollupJobResponse = entityAsMap(client().performRequest(getRollupJobRequest)); Map job = getJob(getRollupJobResponse, rollupJob); assertNotNull(job); @@ -672,12 +741,8 @@ private void assertRollUpJob(final String rollupJob) throws Exception { private void waitForRollUpJob(final String rollupJob, final Matcher expectedStates) throws Exception { assertBusy(() -> { - final Request getRollupJobRequest; - if (isRunningAgainstOldCluster()) { - getRollupJobRequest = new Request("GET", "/_xpack/rollup/job/" + rollupJob); - } else { - getRollupJobRequest = new Request("GET", "/_rollup/job/" + rollupJob); - } + final Request getRollupJobRequest = new Request("GET", getRollupEndpoint() + "/job/" + rollupJob); + Response getRollupJobResponse = client().performRequest(getRollupJobRequest); assertThat(getRollupJobResponse.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));