MC-742 - ElasticSearch - Bugfix - Routing Issue (#1014)

* [MC-724] Corrected the `index_document` action. * [MC-724] Minor code improvements. * [MC-724] Search document routing fix. * [MC-724] Corrected `update_document` action + black formatting. * Added unit additional unit test for search_document with route * [MC-724] Make regenerate * [MC-724] Added `supported_versions` + make regenerate * [MC-742] Remove unused variables and add unit test with no routing. * [MC-742] Applied black formatting. Co-authored-by: PJ Mara <[email protected]> Co-authored-by: Mike Rinehart <[email protected]>
rapid7 · Oct 11, 2021 · 95e938f · 95e938f
1 parent c5aafd6
commit 95e938f
Show file tree

Hide file tree

Showing 11 changed files with 112 additions and 37 deletions.
diff --git a/plugins/elasticsearch/.CHECKSUM b/plugins/elasticsearch/.CHECKSUM
@@ -1,7 +1,7 @@
 {
-	"spec": "97ccaa106810fa6036e9303e00983c02",
-	"manifest": "da98317a75fdc10622f703a90db1dc4c",
-	"setup": "9c3fcd2d9eecb8e8b6d7b7f089fc39b0",
+	"spec": "97158ecd52a2a014e43fb674e57522b0",
+	"manifest": "d907862ad358f8e034ae1280a36a89eb",
+	"setup": "0db39b3c5dd64742886c68ffa85b2f0d",
 	"schemas": [
 		{
 			"identifier": "cluster_health/schema.py",

diff --git a/plugins/elasticsearch/bin/komand_elasticsearch b/plugins/elasticsearch/bin/komand_elasticsearch
@@ -6,7 +6,7 @@ from sys import argv
 
 Name = "Elasticsearch"
 Vendor = "rapid7"
-Version = "3.0.0"
+Version = "3.0.1"
 Description = "Distributed Real-Time Search and Analytics Engine"
 
 

diff --git a/plugins/elasticsearch/help.md b/plugins/elasticsearch/help.md
@@ -14,6 +14,11 @@ This plugin utilizes the [Elasticsearch API](https://www.elastic.co/guide/en/ela
 * An Elasticsearch server
 * Elasticsearch credentials
 
+# Supported Product Versions
+
+* 7.8.1
+* 6.0.0
+
 # Documentation
 
 ## Setup
@@ -379,6 +384,7 @@ _This plugin does not contain any troubleshooting information._
 
 # Version History
 
+* 3.0.1 - Fix issue where Search Documents and Update Documents action returned no results if optional `routing` field was not provided | Update Index Documents action to handle query parameters correctly
 * 3.0.0 - Update to use the `insightconnect-python-3-38-plugin:4` Docker image | Improve error handling | Add `Plugin Exception` | Add `Connection Test` | Add `timeout-decorator` in requirements | Code refactor | Remove input Type from Index Document, Update Document, Search Documents actions and Search Documents trigger | Change inputs name in actions and trigger to not start with `_` | Add `USER nobody` in Dockerfile | Add `api6.py` file for other Elasticsearch version | Add pagination | Add SSL verify
 * 2.0.5 - Updated example inputs and outputs for all the actions
 * 2.0.4 - Correct spelling in help.md

diff --git a/plugins/elasticsearch/komand_elasticsearch/actions/index_document/action.py b/plugins/elasticsearch/komand_elasticsearch/actions/index_document/action.py
@@ -16,28 +16,30 @@ def __init__(self):
         )
 
     def run(self, params={}):
-        clean_params = helper.clean(params)
-        index = clean_params.get(Input.INDEX)
-        id_ = clean_params.get(Input.ID)
-        version = clean_params.get(Input.VERSION)
-        document = clean_params.get(Input.DOCUMENT)
-        parent = clean_params.get(Input.PARENT)
+        version = params.get(Input.VERSION)
+        parent = params.get(Input.PARENT)
 
         query_params = {
-            "version_type": clean_params.get(Input.VERSION_TYPE),
-            "routing": clean_params.get(Input.ROUTING),
-            "timeout": clean_params.get(Input.TIMEOUT),
+            "version_type": params.get(Input.VERSION_TYPE),
+            "routing": params.get(Input.ROUTING),
+            "timeout": params.get(Input.TIMEOUT),
         }
+
         if version:
             query_params["version"] = str(version)
         if parent:
             query_params["parent"] = str(parent)
 
         results = self.connection.client.index(
-            index=index, _id=id_, _type=clean_params.get(Input.TYPE), params=query_params, document=document
+            index=params.get(Input.INDEX),
+            _id=params.get(Input.ID),
+            _type=params.get(Input.TYPE),
+            params=helper.clean(query_params),
+            document=params.get(Input.DOCUMENT),
         )
+
         if results:
-            return {Output.INDEX_RESPONSE: insightconnect_plugin_runtime.helper.clean(results)}
+            return {Output.INDEX_RESPONSE: helper.clean(results)}
 
         raise PluginException(
             cause="Document was not indexed. ", assistance="Please check provided data and try again."

diff --git a/plugins/elasticsearch/komand_elasticsearch/actions/update_document/action.py b/plugins/elasticsearch/komand_elasticsearch/actions/update_document/action.py
@@ -16,20 +16,18 @@ def __init__(self):
         )
 
     def run(self, params={}):
-        clean_params = helper.clean(params)
-        index = clean_params.get(Input.INDEX)
-        id_ = clean_params.get(Input.ID)
-        retry_on_conflict = clean_params.get(Input.RETRY_ON_CONFLICT)
-        wait_for_active_shards = clean_params.get(Input.WAIT_FOR_ACTIVE_SHARDS)
-        version = clean_params.get(Input.VERSION)
+        retry_on_conflict = params.get(Input.RETRY_ON_CONFLICT)
+        wait_for_active_shards = params.get(Input.WAIT_FOR_ACTIVE_SHARDS)
+        version = params.get(Input.VERSION)
 
         query_params = {
-            "refresh": clean_params.get(Input.REFRESH),
-            "source": clean_params.get(Input.SOURCE),
-            "routing": clean_params.get(Input.ROUTING),
-            "parent": clean_params.get(Input.PARENT),
-            "timeout": clean_params.get(Input.TIMEOUT),
+            "refresh": params.get(Input.REFRESH),
+            "source": params.get(Input.SOURCE),
+            "routing": params.get(Input.ROUTING),
+            "parent": params.get(Input.PARENT),
+            "timeout": params.get(Input.TIMEOUT),
         }
+
         if retry_on_conflict:
             query_params["retry_on_conflict"] = str(retry_on_conflict)
         if wait_for_active_shards:
@@ -38,16 +36,16 @@ def run(self, params={}):
             query_params["version"] = str(version)
 
         results = self.connection.client.update(
-            index=index,
-            _id=id_,
-            _type=clean_params.get(Input.TYPE),
-            params=query_params,
-            script=clean_params.get(Input.SCRIPT),
+            index=params.get(Input.INDEX),
+            _id=params.get(Input.ID),
+            _type=params.get(Input.TYPE),
+            params=helper.clean(query_params),
+            script=params.get(Input.SCRIPT),
         )
 
         if not results:
             raise PluginException(
                 cause="Document was not updated", assistance="Please check provided data and try again."
             )
         else:
-            return {Output.UPDATE_RESPONSE: insightconnect_plugin_runtime.helper.clean(results)}
+            return {Output.UPDATE_RESPONSE: helper.clean(results)}
diff --git a/plugins/elasticsearch/komand_elasticsearch/util/request_api.py b/plugins/elasticsearch/komand_elasticsearch/util/request_api.py
@@ -3,6 +3,7 @@
 from logging import Logger
 from insightconnect_plugin_runtime.exceptions import PluginException
 from requests.auth import HTTPBasicAuth
+from insightconnect_plugin_runtime import helper
 
 
 class RequestAPI:
@@ -38,7 +39,10 @@ def _search_first_page(self, path: str, routing: str, scroll_time: str, json_dat
             query = {"query": json_data, "version": "true"}
 
         return self._call_api(
-            method="GET", path=path, params={"routing": routing, "size": 5, "scroll": scroll_time}, json_data=query
+            method="GET",
+            path=path,
+            params=helper.clean({"routing": routing, "size": 5, "scroll": scroll_time}),
+            json_data=query,
         )
 
     def _get_scroll_page(self, scroll_id: str, scroll_time: str):
@@ -58,7 +62,7 @@ def _search_documents(self, path: str, routing: str, json_data: dict = {}) -> di
         elif total is None:
             total = 0
 
-        for i in range(0, 9999):
+        for _ in range(0, 9999):
             if scroll_id:
                 try:
                     scroll_page = self._get_scroll_page(scroll_id, scroll_time)
@@ -68,7 +72,7 @@ def _search_documents(self, path: str, routing: str, json_data: dict = {}) -> di
                     hits.extend(page_hits)
                     took += scroll_page.get("took", 0)
                     scroll_id = scroll_page.get("_scroll_id")
-                except PluginException as e:
+                except PluginException:
                     break
 
         return {

diff --git a/plugins/elasticsearch/plugin.spec.yaml b/plugins/elasticsearch/plugin.spec.yaml
@@ -4,7 +4,8 @@ products: [insightconnect]
 name: elasticsearch
 title: Elasticsearch
 description: Distributed Real-Time Search and Analytics Engine
-version: 3.0.0
+version: 3.0.1
+supported_versions: ["7.8.1", "6.0.0"]
 vendor: rapid7
 support: community
 status: []

diff --git a/plugins/elasticsearch/setup.py b/plugins/elasticsearch/setup.py
@@ -3,7 +3,7 @@
 
 
 setup(name="elasticsearch-rapid7-plugin",
-      version="3.0.0",
+      version="3.0.1",
       description="Distributed Real-Time Search and Analytics Engine",
       author="rapid7",
       author_email="",

diff --git a/plugins/elasticsearch/unit_test/payloads/search_with_route.json.resp b/plugins/elasticsearch/unit_test/payloads/search_with_route.json.resp
@@ -0,0 +1,30 @@
+{
+  "took": 2,
+  "timed_out": false,
+  "_shards": {
+    "total": 1,
+    "successful": 1,
+    "skipped": 0,
+    "failed": 0
+  },
+  "hits": {
+    "total": {
+      "value": 2,
+      "relation": "eq"
+    },
+    "max_score": 1.0,
+    "hits": [
+      {
+        "_index": "test-index",
+        "_type": "_doc",
+        "_id": "VWx5O3oBrBTgS4Hhf6Hp",
+        "_score": 1.0,
+        "_routing": "test-route",
+        "_source": {
+          "id": 1,
+          "message": "Some message"
+        }
+      }
+    ]
+  }
+}
diff --git a/plugins/elasticsearch/unit_test/test_search_documents.py b/plugins/elasticsearch/unit_test/test_search_documents.py
@@ -37,6 +37,26 @@ class TestSearchDocuments(TestCase):
         },
     }
 
+    expected_with_route = {
+        Output.TOOK: 2,
+        Output.TIMED_OUT: False,
+        Output.SHARDS: {"total": 1, "successful": 1, "skipped": 0, "failed": 0},
+        Output.HITS: {
+            "total": {"value": 2},
+            "max_score": 1.0,
+            "hits": [
+                {
+                    "_index": "test-index",
+                    "_type": "_doc",
+                    "_id": "VWx5O3oBrBTgS4Hhf6Hp",
+                    "_score": 1.0,
+                    "_routing": "test-route",
+                    "_source": {"id": 1, "message": "Some message"},
+                },
+            ],
+        },
+    }
+
     @classmethod
     @patch("requests.request", side_effect=Util.mocked_requests_get)
     def setUpClass(cls, mock_request) -> None:
@@ -49,8 +69,20 @@ def test_search_documents(self, mock_request):
         )
         self.assertEqual(actual, self.expected)
 
+    @patch("requests.request", side_effect=Util.mocked_requests_get)
+    def test_search_documents_with_route(self, mock_request):
+        actual = self.action.run(
+            {Input.INDEX: "search-with-route", Input.QUERY: {"query": {"match_all": {}}}, Input.ROUTING: "test-route"}
+        )
+        self.assertEqual(actual, self.expected_with_route)
+
     @patch("requests.request", side_effect=Util.mocked_requests_get)
     def test_search_documents_without_route(self, mock_request):
+        actual = self.action.run({Input.INDEX: "search-without-route", Input.QUERY: {"query": {"match_all": {}}}})
+        self.assertEqual(actual, self.expected)
+
+    @patch("requests.request", side_effect=Util.mocked_requests_get)
+    def test_search_documents_with_route_none(self, mock_request):
         actual = self.action.run(
             {Input.INDEX: "search-without-route", Input.QUERY: {"query": {"match_all": {}}}, Input.ROUTING: None}
         )

diff --git a/plugins/elasticsearch/unit_test/util.py b/plugins/elasticsearch/unit_test/util.py
@@ -64,6 +64,8 @@ def json(self):
             return MockResponse("error", 200)
         elif "search-without-route" in args[1]:
             return MockResponse("search_without_route", 200)
+        elif "search-with-route" in args[1]:
+            return MockResponse("search_with_route", 200)
         elif "search/_search" in args[1] or "trigger-index" in args[1]:
             return MockResponse("search_document", 200)
         elif "UpdateError" in args[1]: