Add Cumulative Cardinality agg (and Data Science plugin)

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java

@@ -121,6 +121,10 @@ private XPackSettings() {
     /** Setting for enabling or disabling vectors. Defaults to true. */
     public static final Setting<Boolean> VECTORS_ENABLED = Setting.boolSetting("xpack.vectors.enabled", true, Setting.Property.NodeScope);
 
+    /** Setting for enabling or disabling data science plugin. Defaults to true. */
+    public static final Setting<Boolean> DATA_SCIENCE_ENABLED = Setting.boolSetting("xpack.datascience.enabled",
+        true, Setting.Property.NodeScope);
+
     /*
      * SSL settings. These are the settings that are specifically registered for SSL. Many are private as we do not explicitly use them
      * but instead parse based on a prefix (eg *.ssl.*)

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java

@@ -38,10 +38,11 @@ public class XPackInfoFeatureAction extends ActionType<XPackInfoFeatureResponse>
     public static final XPackInfoFeatureAction VOTING_ONLY = new XPackInfoFeatureAction(XPackField.VOTING_ONLY);
     public static final XPackInfoFeatureAction FROZEN_INDICES = new XPackInfoFeatureAction(XPackField.FROZEN_INDICES);
     public static final XPackInfoFeatureAction SPATIAL = new XPackInfoFeatureAction(XPackField.SPATIAL);
+    public static final XPackInfoFeatureAction DATA_SCIENCE = new XPackInfoFeatureAction(XPackField.DATA_SCIENCE);
 
     public static final List<XPackInfoFeatureAction> ALL = Arrays.asList(
         SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, SQL, ROLLUP, INDEX_LIFECYCLE, CCR, DATA_FRAME, FLATTENED,
-        VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL
+        VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, DATA_SCIENCE
     );
 
     private XPackInfoFeatureAction(String name) {

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java

@@ -38,10 +38,11 @@ public class XPackUsageFeatureAction extends ActionType<XPackUsageFeatureRespons
     public static final XPackUsageFeatureAction VOTING_ONLY = new XPackUsageFeatureAction(XPackField.VOTING_ONLY);
     public static final XPackUsageFeatureAction FROZEN_INDICES = new XPackUsageFeatureAction(XPackField.FROZEN_INDICES);
     public static final XPackUsageFeatureAction SPATIAL = new XPackUsageFeatureAction(XPackField.SPATIAL);
+    public static final XPackUsageFeatureAction DATA_SCIENCE = new XPackUsageFeatureAction(XPackField.DATA_SCIENCE);
 
     public static final List<XPackUsageFeatureAction> ALL = Arrays.asList(
         SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, SQL, ROLLUP, INDEX_LIFECYCLE, CCR, DATA_FRAME, FLATTENED,
-        VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL
+        VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, DATA_SCIENCE
     );
 
     private XPackUsageFeatureAction(String name) {

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datascience/DataScienceFeatureSetUsage.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.datascience;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.core.XPackFeatureSet;
+import org.elasticsearch.xpack.core.XPackField;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class DataScienceFeatureSetUsage extends XPackFeatureSet.Usage {
+
+    public DataScienceFeatureSetUsage(boolean available, boolean enabled) {
+        super(XPackField.DATA_SCIENCE, available, enabled);
+    }
+
+    public DataScienceFeatureSetUsage(StreamInput input) throws IOException {
+        super(input);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(available, enabled);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        DataScienceFeatureSetUsage other = (DataScienceFeatureSetUsage) obj;
+        return Objects.equals(available, other.available) &&
+            Objects.equals(enabled, other.enabled);
+    }
+}

x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataScienceInfoTransportAction.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.datascience;
+
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.XPackField;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureTransportAction;
+
+public class DataScienceInfoTransportAction extends XPackInfoFeatureTransportAction {
+
+    private final boolean enabled;
+    private final XPackLicenseState licenseState;
+
+    @Inject
+    public DataScienceInfoTransportAction(TransportService transportService, ActionFilters actionFilters,
+                                     Settings settings, XPackLicenseState licenseState) {
+        super(XPackInfoFeatureAction.DATA_SCIENCE.name(), transportService, actionFilters);
+        this.enabled = XPackSettings.DATA_SCIENCE_ENABLED.get(settings);
+        this.licenseState = licenseState;
+    }
+
+    @Override
+    public String name() {
+        return XPackField.DATA_SCIENCE;
+    }
+
+    @Override
+    public boolean available() {
+        return licenseState.isDataScienceAllowed();
+    }
+
+    @Override
+    public boolean enabled() {
+        return enabled;
+    }
+
+}

x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataSciencePlugin.java

@@ -5,17 +5,23 @@
  */
 package org.elasticsearch.xpack.datascience;
 
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.SearchPlugin;
 import org.elasticsearch.xpack.core.XPackPlugin;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
 import org.elasticsearch.xpack.datascience.cumulativecardinality.CumulativeCardinalityPipelineAggregationBuilder;
 import org.elasticsearch.xpack.datascience.cumulativecardinality.CumulativeCardinalityPipelineAggregator;
 
+import java.util.Arrays;
 import java.util.List;
 
 import static java.util.Collections.singletonList;
 
-public class DataSciencePlugin extends Plugin implements SearchPlugin {
+public class DataSciencePlugin extends Plugin implements SearchPlugin, ActionPlugin {
 
     // volatile so all threads can see changes
     protected static volatile boolean isDataScienceAllowed;
@@ -47,4 +53,11 @@ public List<PipelineAggregationSpec> getPipelineAggregations() {
             CumulativeCardinalityPipelineAggregator::new,
             CumulativeCardinalityPipelineAggregationBuilder::parse));
     }
+
+    @Override
+    public List<ActionPlugin.ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
+        return Arrays.asList(
+            new ActionPlugin.ActionHandler<>(XPackUsageFeatureAction.DATA_SCIENCE, DataScienceUsageTransportAction.class),
+            new ActionPlugin.ActionHandler<>(XPackInfoFeatureAction.DATA_SCIENCE, DataScienceInfoTransportAction.class));
+    }
 }

x-pack/plugin/data-science/src/main/java/org/elasticsearch/xpack/datascience/DataScienceUsageTransportAction.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.datascience;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction;
+import org.elasticsearch.xpack.core.datascience.DataScienceFeatureSetUsage;
+
+public class DataScienceUsageTransportAction extends XPackUsageFeatureTransportAction {
+    private final Settings settings;
+    private final XPackLicenseState licenseState;
+
+    @Inject
+    public DataScienceUsageTransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
+                                      ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                      Settings settings, XPackLicenseState licenseState) {
+        super(XPackUsageFeatureAction.DATA_SCIENCE.name(), transportService, clusterService,
+            threadPool, actionFilters, indexNameExpressionResolver);
+        this.settings = settings;
+        this.licenseState = licenseState;
+    }
+
+    @Override
+    protected void masterOperation(Task task, XPackUsageRequest request, ClusterState state,
+                                   ActionListener<XPackUsageFeatureResponse> listener) {
+        boolean available = licenseState.isDataScienceAllowed();
+
+        DataScienceFeatureSetUsage usage =
+            new DataScienceFeatureSetUsage(available, XPackSettings.DATA_SCIENCE_ENABLED.get(settings));
+        listener.onResponse(new XPackUsageFeatureResponse(usage));
+    }
+}

x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/DataScienceInfoTransportActionTests.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.datascience;
+
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.XPackFeatureSet;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
+import org.elasticsearch.xpack.core.datascience.DataScienceFeatureSetUsage;
+import org.junit.Before;
+
+import static org.hamcrest.core.Is.is;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class DataScienceInfoTransportActionTests extends ESTestCase {
+
+    private XPackLicenseState licenseState;
+
+    @Before
+    public void init() {
+        licenseState = mock(XPackLicenseState.class);
+    }
+
+    public void testAvailable() throws Exception {
+        DataScienceInfoTransportAction featureSet = new DataScienceInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), Settings.EMPTY, licenseState);
+        boolean available = randomBoolean();
+        when(licenseState.isDataScienceAllowed()).thenReturn(available);
+        assertThat(featureSet.available(), is(available));
+
+        DataScienceUsageTransportAction usageAction = new DataScienceUsageTransportAction(mock(TransportService.class), null, null,
+            mock(ActionFilters.class), null, Settings.EMPTY, licenseState);
+        PlainActionFuture<XPackUsageFeatureResponse> future = new PlainActionFuture<>();
+        usageAction.masterOperation(null, null, null, future);
+        XPackFeatureSet.Usage usage = future.get().getUsage();
+        assertThat(usage.available(), is(available));
+
+        BytesStreamOutput out = new BytesStreamOutput();
+        usage.writeTo(out);
+        XPackFeatureSet.Usage serializedUsage = new DataScienceFeatureSetUsage(out.bytes().streamInput());
+        assertThat(serializedUsage.available(), is(available));
+    }
+
+    public void testEnabled() throws Exception {
+        Settings.Builder settings = Settings.builder();
+        DataScienceInfoTransportAction featureSet = new DataScienceInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), settings.build(), licenseState);
+        assertThat(featureSet.enabled(), is(true));
+        assertTrue(featureSet.enabled());
+
+        DataScienceUsageTransportAction usageAction = new DataScienceUsageTransportAction(mock(TransportService.class),
+            null, null, mock(ActionFilters.class), null, settings.build(), licenseState);
+        PlainActionFuture<XPackUsageFeatureResponse> future = new PlainActionFuture<>();
+        usageAction.masterOperation(null, null, null, future);
+        XPackFeatureSet.Usage usage = future.get().getUsage();
+        assertTrue(usage.enabled());
+
+        BytesStreamOutput out = new BytesStreamOutput();
+        usage.writeTo(out);
+        XPackFeatureSet.Usage serializedUsage = new DataScienceFeatureSetUsage(out.bytes().streamInput());
+        assertTrue(serializedUsage.enabled());
+    }
+
+}

x-pack/plugin/data-science/src/test/java/org/elasticsearch/datascience/TestAggregatorFactory.java → x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/TestAggregatorFactory.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.datascience;
+package org.elasticsearch.xpack.datascience;
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;

x-pack/plugin/data-science/src/test/java/org/elasticsearch/datascience/cumulativecardinality/CumulativeCardinalityAggregatorTests.java → x-pack/plugin/data-science/src/test/java/org/elasticsearch/xpack/datascience/cumulativecardinality/CumulativeCardinalityAggregatorTests.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.datascience.cumulativecardinality;
+package org.elasticsearch.xpack.datascience.cumulativecardinality;
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.NumericDocValuesField;
@@ -18,7 +18,7 @@
 import org.elasticsearch.common.CheckedConsumer;
 import org.elasticsearch.common.Rounding;
 import org.elasticsearch.common.time.DateFormatters;
-import org.elasticsearch.datascience.TestAggregatorFactory;
+import org.elasticsearch.xpack.datascience.TestAggregatorFactory;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
@@ -46,8 +46,6 @@
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.xpack.datascience.DataSciencePlugin;
-import org.elasticsearch.xpack.datascience.cumulativecardinality.CumulativeCardinalityPipelineAggregationBuilder;
-import org.elasticsearch.xpack.datascience.cumulativecardinality.InternalSimpleLongValue;
 
 import java.io.IOException;
 import java.util.Arrays;