Merge pull request #6857 from BigFunger/add-data-processors-join

[add data] ingest join processor

src/plugins/kibana/public/settings/sections/indices/add_data_steps/pipeline_setup/directives/processor_ui.js

@@ -5,4 +5,5 @@ import './processor_ui_date';
 import './processor_ui_geoip';
 import './processor_ui_grok';
 import './processor_ui_gsub';
+import './processor_ui_join';
 import './processor_ui_set';

src/plugins/kibana/public/settings/sections/indices/add_data_steps/pipeline_setup/directives/processor_ui_join.js

@@ -0,0 +1,41 @@
+import uiModules from 'ui/modules';
+import _ from 'lodash';
+import keysDeep from '../lib/keys_deep';
+import template from '../views/processor_ui_join.html';
+
+const app = uiModules.get('kibana');
+
+//scope.processor, scope.pipeline are attached by the process_container.
+app.directive('processorUiJoin', function () {
+  return {
+    restrict: 'E',
+    template: template,
+    controller : function ($scope) {
+      const processor = $scope.processor;
+      const pipeline = $scope.pipeline;
+
+      function consumeNewInputObject() {
+        const allKeys = keysDeep(processor.inputObject);
+        $scope.fields = _.filter(allKeys, (key) => { return _.isArray(_.get(processor.inputObject, key)); });
+        refreshFieldData();
+      }
+
+      function refreshFieldData() {
+        $scope.fieldData = _.get(processor.inputObject, processor.sourceField);
+      }
+
+      function processorUiChanged() {
+        pipeline.setDirty();
+      }
+
+      $scope.$watch('processor.inputObject', consumeNewInputObject);
+
+      $scope.$watch('processor.sourceField', () => {
+        refreshFieldData();
+        processorUiChanged();
+      });
+
+      $scope.$watch('processor.separator', processorUiChanged);
+    }
+  };
+});

src/plugins/kibana/public/settings/sections/indices/add_data_steps/pipeline_setup/lib/processor_types.js

@@ -182,6 +182,29 @@ export class Gsub extends Processor {
   }
 };
 
+export class Join extends Processor {
+  constructor(processorId) {
+    super(processorId, 'join', 'Join');
+    this.sourceField = '';
+    this.separator = '';
+  }
+
+  get description() {
+    const source = this.sourceField || '?';
+    const separator = this.separator ? ` on '${this.separator}'` : '';
+    return `[${source}]${separator}`;
+  }
+
+  get model() {
+    return {
+      processorId: this.processorId,
+      typeId: this.typeId,
+      sourceField: this.sourceField || '',
+      separator: this.separator || ''
+    };
+  }
+};
+
 export class Set extends Processor {
   constructor(processorId) {
     super(processorId, 'set', 'Set');

src/plugins/kibana/public/settings/sections/indices/add_data_steps/pipeline_setup/views/processor_ui_join.html

@@ -0,0 +1,16 @@
+<div class="form-group">
+  <label>Array Field:</label>
+  <select
+    class="form-control"
+    ng-options="field as field for field in fields"
+    ng-model="processor.sourceField">
+  </select>
+</div>
+<div class="form-group">
+  <label>Field Data:</label>
+  <pre>{{ fieldData }}</pre>
+</div>
+<div class="form-group">
+  <label>Separator:</label>
+  <input type="text" class="form-control" ng-trim="false" ng-model="processor.separator">
+</div>

src/plugins/kibana/server/lib/converters/ingest_processor_api_kibana_to_es_converters.js

@@ -98,6 +98,16 @@ export function gsub(processorApiDocument) {
   };
 }
 
+export function join(processorApiDocument) {
+  return {
+    join: {
+      tag: processorApiDocument.processor_id,
+      field: processorApiDocument.source_field,
+      separator: processorApiDocument.separator
+    }
+  };
+}
+
 export function set(processorApiDocument) {
   return {
     set: {

src/plugins/kibana/server/lib/schemas/resources/ingest_processor_schemas.js

@@ -48,6 +48,12 @@ export const gsub = base.keys({
   replacement: Joi.string().allow('')
 });
 
+export const join = base.keys({
+  type_id: Joi.string().only('join').required(),
+  source_field: Joi.string().allow(''),
+  separator: Joi.string().allow('')
+});
+
 export const set = base.keys({
   type_id: Joi.string().only('set').required(),
   target_field: Joi.string().allow(''),

test/unit/api/ingest/processors/_join.js

@@ -0,0 +1,67 @@
+define(function (require) {
+  var Promise = require('bluebird');
+  var _ = require('intern/dojo/node!lodash');
+  var expect = require('intern/dojo/node!expect.js');
+
+  const testPipeline = {
+    processors: [{
+      processor_id: 'processor1',
+      type_id: 'join',
+      source_field: 'foo',
+      separator: ' '
+    }],
+    input: { foo: ['value1', 'value2'] }
+  };
+
+  return function (bdd, scenarioManager, request) {
+    bdd.describe('simulate - join processor', () => {
+
+      bdd.it('should return 400 for an invalid payload', () => {
+        return Promise.all([
+          // processor requires source_field property
+          request.post('/kibana/ingest/simulate')
+          .send({
+            processors: [{
+              processor_id: 'processor1',
+              type_id: 'join',
+              source_field: 1234,
+              separator: ' '
+            }],
+            input: { foo: ['value1', 'value2'] }
+          })
+          .expect(400)
+        ]);
+      });
+
+      bdd.it('should return 200 for a valid simulate request', () => {
+        return request.post('/kibana/ingest/simulate')
+          .send(testPipeline)
+          .expect(200);
+      });
+
+      bdd.it('should return a simulated output with the correct result for the given processor', function () {
+        return request.post('/kibana/ingest/simulate')
+          .send(testPipeline)
+          .expect(200)
+          .then(function (response) {
+            expect(response.body[0].output.foo).to.be('value1 value2');
+          });
+      });
+
+      bdd.it('should enforce snake case', () => {
+        return request.post('/kibana/ingest/simulate')
+        .send({
+          processors: [{
+            processorId: 'processor1',
+            typeId: 'join',
+            sourceField: 'foo',
+            separator: ' '
+          }],
+          input: { foo: ['value1', 'value2'] }
+        })
+        .expect(400);
+      });
+
+    });
+  };
+});

test/unit/api/ingest/processors/index.js

@@ -5,6 +5,7 @@ define(function (require) {
   var geoip = require('./_geoip');
   var grok = require('./_grok');
   var gsub = require('./_gsub');
+  var join = require('./_join');
   var set = require('./_set');
 
   return function processors(bdd, scenarioManager, request) {
@@ -14,6 +15,7 @@ define(function (require) {
     geoip(bdd, scenarioManager, request);
     grok(bdd, scenarioManager, request);
     gsub(bdd, scenarioManager, request);
+    join(bdd, scenarioManager, request);
     set(bdd, scenarioManager, request);
   };