From f6ade3a20c5ab69fb18e18d5f3a9af453b47e84d Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Mon, 4 Feb 2019 13:06:57 -0700 Subject: [PATCH] Dep. check for ECS changes to User Agent processor The User Agent ingest processor is changing to align with ECS. Users need to be made aware that any pipelines using this processor will behave differently in 7.0. --- .../elasticsearch/ingest/IngestService.java | 2 +- .../deprecation/ClusterDeprecationChecks.java | 40 +++++++++++++ .../xpack/deprecation/DeprecationChecks.java | 1 + .../ClusterDeprecationChecksTests.java | 57 +++++++++++++++++++ 4 files changed, 99 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index 1031308d7a70a..c55724d189947 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -335,7 +335,7 @@ public String getType() { return new Pipeline(id, description, null, new CompoundProcessor(failureProcessor)); } - static ClusterState innerPut(PutPipelineRequest request, ClusterState currentState) { + public static ClusterState innerPut(PutPipelineRequest request, ClusterState currentState) { IngestMetadata currentIngestMetadata = currentState.metaData().custom(IngestMetadata.TYPE); Map pipelines; if (currentIngestMetadata != null) { diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java index 28ff1b2673f70..60ba41c06fb5f 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java @@ -8,8 +8,16 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.ingest.ConfigurationUtils; +import org.elasticsearch.ingest.IngestService; +import org.elasticsearch.ingest.PipelineConfiguration; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + public class ClusterDeprecationChecks { static DeprecationIssue checkShardLimit(ClusterState state) { @@ -40,4 +48,36 @@ static DeprecationIssue checkClusterName(ClusterState state) { } return null; } + + @SuppressWarnings("unchecked") + static DeprecationIssue checkUserAgentPipelines(ClusterState state) { + List pipelines = IngestService.getPipelines(state); + + List pipelinesWithDeprecatedEcsConfig = pipelines.stream() + .filter(Objects::nonNull) + .filter(pipeline -> { + Map pipelineConfig = pipeline.getConfigAsMap(); + + List>> processors = + (List>>) pipelineConfig.get("processors"); + return processors.stream() + .filter(Objects::nonNull) + .filter(processor -> processor.containsKey("user_agent")) + .map(processor -> processor.get("user_agent")) + .anyMatch(processorConfig -> + false == ConfigurationUtils.readBooleanProperty(null, null, processorConfig, "ecs", false)); + }) + .map(PipelineConfiguration::getId) + .sorted() // Make the warning consistent for testing purposes + .collect(Collectors.toList()); + if (pipelinesWithDeprecatedEcsConfig.isEmpty() == false) { + return new DeprecationIssue(DeprecationIssue.Level.WARNING, + "User-Agent ingest plugin will use ECS-formatted output", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + + "#ingest-user-agent-ecs-always", + "Ingest pipelines " + pipelinesWithDeprecatedEcsConfig + " will change to using ECS output format in 7.0"); + } + return null; + + } } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java index 6a22f694771dc..d48f735ef0d5a 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java @@ -33,6 +33,7 @@ private DeprecationChecks() { static List> CLUSTER_SETTINGS_CHECKS = Collections.unmodifiableList(Arrays.asList( + ClusterDeprecationChecks::checkUserAgentPipelines, ClusterDeprecationChecks::checkShardLimit, ClusterDeprecationChecks::checkClusterName )); diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java index f2d407b024e1a..09bc2b991fa8a 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java @@ -6,14 +6,18 @@ package org.elasticsearch.xpack.deprecation; import org.elasticsearch.Version; +import org.elasticsearch.action.ingest.PutPipelineRequest; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.ingest.IngestService; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; @@ -83,4 +87,57 @@ public void testCheckShardLimit() { issues = DeprecationChecks.filterChecks(CLUSTER_SETTINGS_CHECKS, c -> c.apply(goodState)); assertTrue(issues.isEmpty()); } + + public void testUserAgentEcsCheck() { + PutPipelineRequest ecsFalseRequest = new PutPipelineRequest("ecs_false", + new BytesArray("{\n" + + " \"description\" : \"This has ecs set to false\",\n" + + " \"processors\" : [\n" + + " {\n" + + " \"user_agent\" : {\n" + + " \"field\" : \"agent\",\n" + + " \"ecs\" : false\n" + + " }\n" + + " }\n" + + " ]\n" + + "}"), XContentType.JSON); + PutPipelineRequest ecsNullRequest = new PutPipelineRequest("ecs_null", + new BytesArray("{\n" + + " \"description\" : \"This has ecs set to false\",\n" + + " \"processors\" : [\n" + + " {\n" + + " \"user_agent\" : {\n" + + " \"field\" : \"agent\"\n" + + " }\n" + + " }\n" + + " ]\n" + + "}"), XContentType.JSON); + PutPipelineRequest ecsTrueRequest = new PutPipelineRequest("ecs_true", + new BytesArray("{\n" + + " \"description\" : \"This has ecs set to false\",\n" + + " \"processors\" : [\n" + + " {\n" + + " \"user_agent\" : {\n" + + " \"field\" : \"agent\",\n" + + " \"ecs\" : true\n" + + " }\n" + + " }\n" + + " ]\n" + + "}"), XContentType.JSON); + + ClusterState state = ClusterState.builder(new ClusterName("test")).build(); + state = IngestService.innerPut(ecsTrueRequest, state); + state = IngestService.innerPut(ecsFalseRequest, state); + state = IngestService.innerPut(ecsNullRequest, state); + + final ClusterState finalState = state; + List issues = DeprecationChecks.filterChecks(CLUSTER_SETTINGS_CHECKS, c -> c.apply(finalState)); + + DeprecationIssue expected = new DeprecationIssue(DeprecationIssue.Level.WARNING, + "User-Agent ingest plugin will use ECS-formatted output", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + + "#ingest-user-agent-ecs-always", + "Ingest pipelines [ecs_false, ecs_null] will change to using ECS output format in 7.0"); + assertEquals(singletonList(expected), issues); + } }