From 2b9023f8ec8f36a561249fef9db334fe84f828bd Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 19 May 2021 10:07:11 +0200 Subject: [PATCH 1/2] The get aliases api should not return entries for data streams with no aliases Backport #72953 to 7.x branch. The get alias api should take into account the aliases parameter when returning aliases that refer to data streams and don't return entries for data streams that don't have any aliases pointing to it. Relates to #66163 --- docs/reference/indices/aliases.asciidoc | 3 -- .../alias/get/TransportGetAliasesAction.java | 13 ++++--- .../cluster/metadata/Metadata.java | 12 +++--- .../get/TransportGetAliasesActionTests.java | 37 +++++++++++++++++++ .../xpack/datastreams/DataStreamsRestIT.java | 13 ++----- 5 files changed, 55 insertions(+), 23 deletions(-) diff --git a/docs/reference/indices/aliases.asciidoc b/docs/reference/indices/aliases.asciidoc index 01dd4d5452cf1..31b56c5e8cba1 100644 --- a/docs/reference/indices/aliases.asciidoc +++ b/docs/reference/indices/aliases.asciidoc @@ -599,9 +599,6 @@ The get index alias API returns: [source,console-result] ---- { - "logs-my_app-default": { - "aliases": {} - }, "logs-nginx.access-prod": { "aliases": { "logs": {} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java index e4273fa032012..fa67735b8569e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java @@ -72,7 +72,8 @@ protected void masterOperation(GetAliasesRequest request, ClusterState state, Ac final SystemIndexAccessLevel systemIndexAccessLevel = indexNameExpressionResolver.getSystemIndexAccessLevel(); ImmutableOpenMap> aliases = state.metadata().findAliases(request, concreteIndices); listener.onResponse(new GetAliasesResponse(postProcess(request, concreteIndices, aliases, state, - systemIndexAccessLevel, threadPool.getThreadContext(), systemIndices), postProcess(request, state))); + systemIndexAccessLevel, threadPool.getThreadContext(), systemIndices), + postProcess(indexNameExpressionResolver, request, state))); } /** @@ -106,17 +107,19 @@ static ImmutableOpenMap> postProcess(GetAliasesReque return finalResponse; } - Map> postProcess(GetAliasesRequest request, ClusterState state) { + static Map> postProcess(IndexNameExpressionResolver resolver, GetAliasesRequest request, + ClusterState state) { Map> result = new HashMap<>(); boolean noAliasesSpecified = request.getOriginalAliases() == null || request.getOriginalAliases().length == 0; - List requestedDataStreams = - indexNameExpressionResolver.dataStreamNames(state, request.indicesOptions(), request.indices()); + List requestedDataStreams = resolver.dataStreamNames(state, request.indicesOptions(), request.indices()); for (String requestedDataStream : requestedDataStreams) { List aliases = state.metadata().dataStreamAliases().values().stream() .filter(alias -> alias.getDataStreams().contains(requestedDataStream)) .filter(alias -> noAliasesSpecified || Regex.simpleMatch(request.aliases(), alias.getName())) .collect(Collectors.toList()); - result.put(requestedDataStream, aliases); + if (aliases.isEmpty() == false) { + result.put(requestedDataStream, aliases); + } } return result; } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java index 5aeb5999aa388..9653aa0ba9252 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java @@ -1300,7 +1300,7 @@ public Builder put(DataStream dataStream) { return this; } - public boolean put(String name, String dataStream) { + public boolean put(String aliasName, String dataStream) { Map existingDataStream = Optional.ofNullable((DataStreamMetadata) this.customs.get(DataStreamMetadata.TYPE)) .map(dsmd -> new HashMap<>(dsmd.dataStreams())) @@ -1311,21 +1311,21 @@ public boolean put(String name, String dataStream) { .orElse(new HashMap<>()); if (existingDataStream.containsKey(dataStream) == false) { - throw new IllegalArgumentException("alias [" + name + "] refers to a non existing data stream [" + dataStream + "]"); + throw new IllegalArgumentException("alias [" + aliasName + "] refers to a non existing data stream [" + dataStream + "]"); } - DataStreamAlias alias = dataStreamAliases.get(name); + DataStreamAlias alias = dataStreamAliases.get(aliasName); if (alias == null) { - alias = new DataStreamAlias(name, Collections.singletonList(dataStream)); + alias = new DataStreamAlias(aliasName, Collections.singletonList(dataStream)); } else { Set dataStreams = new HashSet<>(alias.getDataStreams()); boolean added = dataStreams.add(dataStream); if (added == false) { return false; } - alias = new DataStreamAlias(name, dataStreams); + alias = new DataStreamAlias(aliasName, dataStreams); } - dataStreamAliases.put(name, alias); + dataStreamAliases.put(aliasName, alias); this.customs.put(DataStreamMetadata.TYPE, new DataStreamMetadata(existingDataStream, dataStreamAliases)); return true; diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java index 229de26d68713..5c4a5ba4babf6 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java @@ -10,20 +10,26 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.cluster.metadata.DataStreamAlias; +import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.indices.EmptySystemIndices; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel; +import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.test.ESTestCase; import java.util.Collections; import java.util.List; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; public class TransportGetAliasesActionTests extends ESTestCase { @@ -184,6 +190,37 @@ public void testDeprecationWarningEmittedWhenRequestingNonExistingAliasInSystemP "access to system indices and their aliases will not be allowed"); } + public void testPostProcessDataStreamAliases() { + var resolver = TestIndexNameExpressionResolver.newInstance(); + var tuples = List.of(new Tuple<>("logs-foo", 1), new Tuple<>("logs-bar", 1), new Tuple<>("logs-baz", 1)); + var clusterState = DataStreamTestHelper.getClusterStateWithDataStreams(tuples, List.of()); + var builder = Metadata.builder(clusterState.metadata()); + builder.put("logs", "logs-foo"); + builder.put("logs", "logs-bar"); + builder.put("secret", "logs-bar"); + clusterState = ClusterState.builder(clusterState).metadata(builder).build(); + + // return all all data streams with aliases + var getAliasesRequest = new GetAliasesRequest(); + var result = TransportGetAliasesAction.postProcess(resolver, getAliasesRequest, clusterState); + assertThat(result.keySet(), containsInAnyOrder("logs-foo", "logs-bar")); + assertThat(result.get("logs-foo"), contains(new DataStreamAlias("logs", List.of("logs-bar", "logs-foo")))); + assertThat(result.get("logs-bar"), containsInAnyOrder(new DataStreamAlias("logs", List.of("logs-bar", "logs-foo")), + new DataStreamAlias("secret", List.of("logs-bar")))); + + // filter by alias name + getAliasesRequest = new GetAliasesRequest("secret"); + result = TransportGetAliasesAction.postProcess(resolver, getAliasesRequest, clusterState); + assertThat(result.keySet(), containsInAnyOrder("logs-bar")); + assertThat(result.get("logs-bar"), contains(new DataStreamAlias("secret", List.of("logs-bar")))); + + // filter by data stream: + getAliasesRequest = new GetAliasesRequest().indices("logs-foo"); + result = TransportGetAliasesAction.postProcess(resolver, getAliasesRequest, clusterState); + assertThat(result.keySet(), containsInAnyOrder("logs-foo")); + assertThat(result.get("logs-foo"), contains(new DataStreamAlias("logs", List.of("logs-bar", "logs-foo")))); + } + public ClusterState systemIndexTestClusterState() { return ClusterState.builder(ClusterState.EMPTY_STATE) .metadata(Metadata.builder() diff --git a/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java index 056354543dd0a..3e4a1afe15cfa 100644 --- a/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java +++ b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java @@ -132,8 +132,7 @@ public void testDataStreamAliases() throws Exception { getAliasesRequest = new Request("GET", "/_aliases"); getAliasesResponse = entityAsMap(client().performRequest(getAliasesRequest)); - assertEquals(emptyMap(), XContentMapValues.extractValue("logs-myapp1.aliases", getAliasesResponse)); - assertEquals(emptyMap(), XContentMapValues.extractValue("logs-myapp2.aliases", getAliasesResponse)); + assertEquals(emptyMap(), getAliasesResponse); expectThrows(ResponseException.class, () -> client().performRequest(new Request("GET", "/logs/_search"))); // Add logs-* -> logs @@ -157,8 +156,7 @@ public void testDataStreamAliases() throws Exception { getAliasesRequest = new Request("GET", "/_aliases"); getAliasesResponse = entityAsMap(client().performRequest(getAliasesRequest)); - assertEquals(emptyMap(), XContentMapValues.extractValue("logs-myapp1.aliases", getAliasesResponse)); - assertEquals(emptyMap(), XContentMapValues.extractValue("logs-myapp2.aliases", getAliasesResponse)); + assertEquals(emptyMap(), getAliasesResponse); expectThrows(ResponseException.class, () -> client().performRequest(new Request("GET", "/logs/_search"))); } @@ -227,20 +225,17 @@ public void testGetAliasApiFilterByDataStreamAlias() throws Exception { response = client().performRequest(new Request("GET", "/_alias/emea")); assertOK(response); getAliasesResponse = entityAsMap(response); - assertThat(getAliasesResponse.size(), equalTo(2)); // Adjust to equalTo(1) when #72953 is merged + assertThat(getAliasesResponse.size(), equalTo(1)); assertEquals(singletonMap("emea", emptyMap()), XContentMapValues.extractValue("logs-emea.aliases", getAliasesResponse)); - assertEquals(emptyMap(), XContentMapValues.extractValue("logs-nasa.aliases", getAliasesResponse)); // Remove when #72953 is merged ResponseException exception = expectThrows(ResponseException.class, () -> client().performRequest(new Request("GET", "/_alias/wrong_name"))); response = exception.getResponse(); assertThat(response.getStatusLine().getStatusCode(), equalTo(404)); getAliasesResponse = entityAsMap(response); - assertThat(getAliasesResponse.size(), equalTo(4)); // Adjust to equalTo(2) when #72953 is merged + assertThat(getAliasesResponse.size(), equalTo(2)); assertEquals("alias [wrong_name] missing", getAliasesResponse.get("error")); assertEquals(404, getAliasesResponse.get("status")); - assertEquals(emptyMap(), XContentMapValues.extractValue("logs-emea.aliases", getAliasesResponse)); // Remove when #72953 is merged - assertEquals(emptyMap(), XContentMapValues.extractValue("logs-nasa.aliases", getAliasesResponse)); // Remove when #72953 is merged } } From 3e780b16ff4530c73a52be9c929de65361310248 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 19 May 2021 10:50:22 +0200 Subject: [PATCH 2/2] adjust test to 7.x branch --- .../get/TransportGetAliasesActionTests.java | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java index 5c4a5ba4babf6..32f81a4d0368e 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.cluster.metadata.DataStreamAlias; import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.Tuple; @@ -25,8 +26,10 @@ import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.test.ESTestCase; +import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; @@ -191,34 +194,35 @@ public void testDeprecationWarningEmittedWhenRequestingNonExistingAliasInSystemP } public void testPostProcessDataStreamAliases() { - var resolver = TestIndexNameExpressionResolver.newInstance(); - var tuples = List.of(new Tuple<>("logs-foo", 1), new Tuple<>("logs-bar", 1), new Tuple<>("logs-baz", 1)); - var clusterState = DataStreamTestHelper.getClusterStateWithDataStreams(tuples, List.of()); - var builder = Metadata.builder(clusterState.metadata()); + IndexNameExpressionResolver resolver = TestIndexNameExpressionResolver.newInstance(); + List> tuples = + Arrays.asList(new Tuple<>("logs-foo", 1), new Tuple<>("logs-bar", 1), new Tuple<>("logs-baz", 1)); + ClusterState clusterState = DataStreamTestHelper.getClusterStateWithDataStreams(tuples, Collections.emptyList()); + Metadata.Builder builder = Metadata.builder(clusterState.metadata()); builder.put("logs", "logs-foo"); builder.put("logs", "logs-bar"); builder.put("secret", "logs-bar"); clusterState = ClusterState.builder(clusterState).metadata(builder).build(); // return all all data streams with aliases - var getAliasesRequest = new GetAliasesRequest(); - var result = TransportGetAliasesAction.postProcess(resolver, getAliasesRequest, clusterState); + GetAliasesRequest getAliasesRequest = new GetAliasesRequest(); + Map> result = TransportGetAliasesAction.postProcess(resolver, getAliasesRequest, clusterState); assertThat(result.keySet(), containsInAnyOrder("logs-foo", "logs-bar")); - assertThat(result.get("logs-foo"), contains(new DataStreamAlias("logs", List.of("logs-bar", "logs-foo")))); - assertThat(result.get("logs-bar"), containsInAnyOrder(new DataStreamAlias("logs", List.of("logs-bar", "logs-foo")), - new DataStreamAlias("secret", List.of("logs-bar")))); + assertThat(result.get("logs-foo"), contains(new DataStreamAlias("logs", Arrays.asList("logs-bar", "logs-foo")))); + assertThat(result.get("logs-bar"), containsInAnyOrder(new DataStreamAlias("logs", Arrays.asList("logs-bar", "logs-foo")), + new DataStreamAlias("secret", Collections.singletonList("logs-bar")))); // filter by alias name getAliasesRequest = new GetAliasesRequest("secret"); result = TransportGetAliasesAction.postProcess(resolver, getAliasesRequest, clusterState); assertThat(result.keySet(), containsInAnyOrder("logs-bar")); - assertThat(result.get("logs-bar"), contains(new DataStreamAlias("secret", List.of("logs-bar")))); + assertThat(result.get("logs-bar"), contains(new DataStreamAlias("secret", Collections.singletonList("logs-bar")))); // filter by data stream: getAliasesRequest = new GetAliasesRequest().indices("logs-foo"); result = TransportGetAliasesAction.postProcess(resolver, getAliasesRequest, clusterState); assertThat(result.keySet(), containsInAnyOrder("logs-foo")); - assertThat(result.get("logs-foo"), contains(new DataStreamAlias("logs", List.of("logs-bar", "logs-foo")))); + assertThat(result.get("logs-foo"), contains(new DataStreamAlias("logs", Arrays.asList("logs-bar", "logs-foo")))); } public ClusterState systemIndexTestClusterState() {