merge from develop, release for 1.1.0

geye-fe/src/components/handleCenter/utils/searchResultItem.vue

@@ -87,7 +87,8 @@
           .then(resp => {
             if (resp.data.code === 1001) {
               // this.hasShow = false;
-              this.$message.success("已确认信息泄露！")
+              this.$message.success("已确认信息泄露！");
+              this.hasShow = false;
             } else {
               this.$message.error(resp.data.message);
             }

geye/core/engine/filter/filter.py

@@ -105,6 +105,12 @@ def get_raw_code(self, raw_code_url):
         return result
 
     def put_task_to_queue(self, task, target_queue: queue.PriorityQueue = None):
+        """
+        把任务放到队列中去
+        :param task: 待处理的任务
+        :param target_queue: 待放入的队列
+        :return:
+        """
         if not target_queue:
             target_queue = self.filter_task_queue
         while self.status == self.EngineStatus.RUNNING:
@@ -117,6 +123,13 @@ def put_task_to_queue(self, task, target_queue: queue.PriorityQueue = None):
 
     @staticmethod
     def __get_content_by_position(position, task, raw_code):
+        """
+        根据position id获取对应位置的内容
+        :param position:
+        :param task:
+        :param raw_code:
+        :return:
+        """
         position = int(position)
         result = {
             "success": True,

geye/core/engine/filter/rule.py

@@ -140,23 +140,48 @@ def regex_filter(rule_content, filter_content, frid) -> dict:
             result_queue = multiprocessing.Queue()
             p = Process(target=RuleEngine._regex_inner_engine, args=(rule_content, filter_content, result_queue, ))
             p.start()
+
+            # 等待60秒来进行正则匹配
+            p.join(60)
+            if p.is_alive():
+                logger.error("[INNER REGEX] filter timeout! frid: {}".format(frid))
+                p.terminate()
+                p.join()
+
+                # 主动释放queue，防止内存泄露
+                del result_queue
+                filter_result["error"] = True
+                return filter_result
+
+            # 获取queue中的数据
             try:
-                p.join(60)
                 _result = result_queue.get_nowait()
                 filter_result["found"] = _result["found"]
                 filter_result["code"] = _result["code"]
                 return filter_result
-            except multiprocessing.TimeoutError:
-                # 线程超时
-                logger.error("[INNER REGEX] filter timeout! frid: {}".format(frid))
-                p.terminate()
-                filter_result["error"] = True
-                return filter_result
             except queue.Empty:
-                # 线程结束了，但是没获取到东西
+                # 进程结束了，但是没获取到东西
                 logger.error("Empty result get from queue! frid: {}".format(frid))
                 filter_result["error"] = True
                 return filter_result
+
+            # try:
+            #     p.join(60)
+            #     _result = result_queue.get_nowait()
+            #     filter_result["found"] = _result["found"]
+            #     filter_result["code"] = _result["code"]
+            #     return filter_result
+            # except multiprocessing.TimeoutError:
+            #     # 进程超时
+            #     logger.error("[INNER REGEX] filter timeout! frid: {}".format(frid))
+            #     p.terminate()
+            #     filter_result["error"] = True
+            #     return filter_result
+            # except queue.Empty:
+            #     # 线程结束了，但是没获取到东西
+            #     logger.error("Empty result get from queue! frid: {}".format(frid))
+            #     filter_result["error"] = True
+            #     return filter_result
         elif settings.REGEX_ENGINE == "grep":
             # grep engine
             rule = shlex.quote(rule_content)
@@ -173,19 +198,38 @@ def regex_filter(rule_content, filter_content, frid) -> dict:
     @staticmethod
     def string_filter(rule_content, filter_content) -> dict:
         """
-        普通的字符串in查找
-        先通过for迭代查找每一行，todo: 后面需要重构，不然会影响效率
+        普通的字符串查找，支持简单的Python表达式，按照行进行查找
+        :param rule_content:
+            规则内容
+            "password" in {{value}} and "token" not in {{value}}
+        :param filter_content: 待过滤的内容
+
+
         """
         result = {
             "error": False,
             "found": False,
             "code": ""
         }
+
+        # TODO: 这里存在代码执行的风险，先简单过滤，实际上并没有什么用
+        if "__class__" in rule_content or "__subclasses__" in rule_content:
+            logger.fatal("危险规则，禁止执行!")
+            return result
+
+        rule_content = rule_content.replace("{{content}}", "line")
+
         # logger.debug("rule_content: {}, filter_content: {}".format(rule_content, filter_content))
+        # eval_result = eval(rule_content, globals={"__builtins__": None}, locals={"filter_content": filter_content})
+        # if not eval_result:
+        #     # 没找到，直接返回result对象即可
+        #     return result
+
+        # 找到了，按照行来找上下文代码
         filter_content_array = RuleEngine.convert_code_to_list(filter_content)
         # logger.debug("filter_content_array: {}".format(filter_content_array))
         for line_no, line in enumerate(filter_content_array):
-            if rule_content in line:
+            if eval(rule_content, {"__builtins__": None}, {"line": line}):
                 result["found"] = True
                 # 取前后各5行代码
                 begin_no, end_no, code_array = RuleEngine.get_neighbor_code(line_no, filter_content_array, 5)

geye/tests/__init__.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+
+"""
+    
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    
+
+    :author:    lightless <[email protected]>
+    :homepage:  None
+    :license:   GPL-3.0, see LICENSE for more details.
+    :copyright: Copyright (c) 2017 lightless. All rights reserved
+"""

geye/tests/test_eval.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+
+"""
+    
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    
+
+    :author:    lightless <[email protected]>
+    :homepage:  None
+    :license:   GPL-3.0, see LICENSE for more details.
+    :copyright: Copyright (c) 2017 lightless. All rights reserved
+"""
+
+import ast
+
+
+# rule_content = "'password' in {{content}} and 'token' not in {{content}}"
+rule_content = '[x for x in {}.__class__.__bases__[0].__subclasses__() if x.__name__ == "zipimporter"]'
+# rule_content = '[].__class__.__mro__[-1].__subclasses__()'
+filter_content = "aaaabbbcccdde\naaacccbdbadjf"
+# filter_content = "aaaabbbcccdde\naaacccbdbadjf\npassword=123"
+
+# print("globals: {}".format(globals()))
+# print("locals: {}".format(locals()))
+
+code = rule_content.replace("{{content}}", "filter_content")
+print("final code: {}".format(code))
+
+result = eval(code, {"__builtins__": None}, {"filter_content": filter_content})
+print(result)
+
+
+# class ASTNode(ast.NodeVisitor):
+#     def generic_visit(self, node):
+#         print("node: {}".format(dir(node.body)))
+#         print("node: {}".format(node.body.keywords))
+#         print("node name: {}".format(type(node)))
+#
+#
+# n = ast.parse(code, '<usercode>', "eval")
+# v = ASTNode()
+# v.visit(n)
+#
+# print("global node: {}".format(n))

geye/tests/test_lua.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+
+"""
+    
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    
+
+    :author:    lightless <[email protected]>
+    :homepage:  None
+    :license:   GPL-3.0, see LICENSE for more details.
+    :copyright: Copyright (c) 2017 lightless. All rights reserved
+"""
+from lupa import LuaRuntime
+
+lua = LuaRuntime()
+lua.eval("1+1")
+
+
+# rule_content = "'password' in {{content}} and 'token' not in {{content}}".replace("\'", "\"")
+rule_content = '[x for x in {}.__class__.__bases__[0].__subclasses__() if x.__name__ == "zipimporter"]'
+# rule_content = '[].__class__.__mro__[-1].__subclasses__()'
+# filter_content = "aaaabbbcccdde\naaacccbdbadjf"
+filter_content = "aaaabbbcccdde\naaacccbdbadjf\npassword=123"
+
+code = rule_content.replace("{{content}}", "filter_content")
+code = "python.eval(\'{code}\')".format(code=code)
+print("final code: {}".format(code))
+
+result = lua.eval(code)
+print("result: {}".format(result))

geye/tests/test_process.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+
+"""
+    
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    
+
+    :author:    lightless <[email protected]>
+    :homepage:  None
+    :license:   GPL-3.0, see LICENSE for more details.
+    :copyright: Copyright (c) 2017 lightless. All rights reserved
+"""
+
+import time
+import multiprocessing
+
+
+def worker():
+    print("start long job!")
+    time.sleep(15)
+    print("end long job!")
+
+
+p = multiprocessing.Process(target=worker)
+p.start()
+
+# try:
+#     print("start join!")
+#     p.join(5)
+#     print("join done!")
+# except multiprocessing.TimeoutError:
+#     print("timeout!")
+#     p.terminate()
+
+# 下面这段是正确的杀掉子进程的方法
+# 确认下queue会不会导致内存泄露
+p.join(5)
+if p.is_alive():
+    print("still alive! kill it!")
+    p.terminate()
+    p.join()
+
+
+print("main done!")
+
+while True:
+    time.sleep(1)