Merge branch 'release/1.13.0'

analyzers/DShield/DShield_lookup.json

@@ -0,0 +1,14 @@
+{
+  "name": "DShield_lookup",
+  "version": "1.0",
+  "author": "Xavier Xavier, SANS ISC",
+  "url": "https://github.com/xme/thehive/Cortex-Analyzers",
+  "license": "AGPL-V3",
+  "description": "Query the SANS ISC DShield API to check for an IP address reputation.",
+  "dataTypeList": ["ip"],
+  "command": "DShield/DShield_lookup.py",
+  "baseConfig": "DShield",
+  "config": {
+      "service": "query"
+  }
+}

analyzers/DShield/DShield_lookup.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+import json
+import requests
+import datetime
+import math
+from cortexutils.analyzer import Analyzer
+
+class DShieldAnalyzer(Analyzer):
+    def __init__(self):
+        Analyzer.__init__(self)
+
+    def dshield_checkip(self, data):
+        url = 'https://isc.sans.edu/api/ip/%s?json' % data
+        r = requests.get(url)
+        return json.loads(r.text)
+
+    def artifacts(self, raw):
+        artifacts = []
+        if 'as' in raw:
+            artifacts.append({'type':'autonomous-system','value':str(raw['as'])})
+
+        if 'asabusecontact' in raw:
+            artifacts.append({'type': 'email', 'value':str(raw['asabusecontact'])})
+        return artifacts
+
+    def summary(self, raw):
+        taxonomies = []
+        value = "-"
+        level = 'safe'
+
+        categories = raw.get("Categories", None)
+        blacklists = raw.get("Blacklists", None)
+        num_categories = raw.get("Categories Identifier", None)
+
+        if 'maxrisk' in raw:
+            if 'threatfeedscount' in raw:
+                r = int(raw['maxrisk']) + raw['threatfeedscount']
+            else:
+                r = int(raw['maxrisk'])
+            if r <= 1:
+                level = 'safe'
+            elif r <= 6:
+                level = 'suspicious'
+            else:
+                level = 'malicious'
+
+        value = "{} count(s) / {} attack(s) / {} threatfeed(s)".format(raw['count'], raw['attacks'], raw['threatfeedscount'])
+
+        taxonomies.append(self.build_taxonomy(level, "DShield", "Score", value))
+        return {"taxonomies": taxonomies}
+
+    def get_reputation(self, risk):
+        if risk <= 1:
+            return("Safe")
+        elif risk <= 6:
+            return("Suspicious")
+        else:
+            return("Malicious")
+
+    def run(self):
+        if self.data_type == 'ip':
+            data = self.get_param('data', None, 'Data is missing')
+            r = self.dshield_checkip(data)
+            # Do we get valid results
+            if self.data_type in r.keys():
+                info = r[self.data_type]
+                results = {}
+                results['ip'] = info['number']
+                results['count'] = info['count'] if isinstance(info['count'], int) else 0
+                results['attacks'] = info['attacks'] if isinstance(info['attacks'], int) else 0
+                results['lastseen'] = info['maxdate'] if isinstance(info['maxdate'], str) else 'None'
+                results['firstseen'] = info['mindate'] if isinstance(info['mindate'], str) else 'None'
+                results['updated'] = info['updated'] if isinstance(info['updated'], str) else 'None'
+                results['comment'] = info['comment'] if isinstance(info['comment'], str) else 'None'
+                results['asabusecontact'] = info['asabusecontact'] if isinstance(info['asabusecontact'], str) else 'Unknown'
+                results['as'] = info['as']
+                results['asname'] = info['asname']
+                results['ascountry'] = info['ascountry']
+                results['assize'] = info['assize']
+                results['network'] = info['network']
+                results['threatfeedscount'] = 0
+                if 'threatfeeds' not in info:
+                    results['threatfeeds'] = ''
+                else:
+                    results['threatfeedscount'] = len(json.loads(json.dumps(info['threatfeeds'])))
+                    results['threatfeeds'] = info['threatfeeds'] 
+                # Compute a risk level based on collected information
+                results['maxrisk'] = 0
+                maxrisk = 10
+                if results['attacks'] > 0:
+                    results['maxrisk'] = round(min(math.log10(results['attacks']) * 2, maxrisk))
+
+                # We add the number of threat feeds to the maxrisk to increase the detection rate
+                results['reputation'] = self.get_reputation(int(results['maxrisk']) + results['threatfeedscount'])
+                self.report(results)
+            else:
+                self.error('No data found')
+        else:
+            self.error('Invalid data type')
+
+
+if __name__ == '__main__':
+    DShieldAnalyzer().run()

analyzers/DShield/requirements.txt

@@ -0,0 +1,2 @@
+cortexutils
+urllib2

analyzers/DomainTools/domaintools_analyzer.py

@@ -78,6 +78,9 @@ def summary(self, raw):
                     "domain_count": sum(d["domain_count"] for d in raw["ip_addresses"])
                 }
 
+        if "record_count" in raw:
+            r["record_count"] = raw.get('record_count')
+
         if "domain_count" in raw:
             r["domain_count"] = {
                 "current": raw["domain_count"]["current"],
@@ -122,7 +125,7 @@ def summary(self, raw):
 
         if r["service"] == "whois/history":
             taxonomies.append(self.build_taxonomy("info", "DT", "Whois_History",
-                                                  "{}, {} domains ".format(r["name_server"], r["domain_count"])))
+                                                  "{} {}".format(r["record_count"], "records" if r["record_count"] > 1 else "record")))
 
         if r["service"] == "whois/parsed" or r['service'] == "whois":
             if r["registrar"]:
@@ -170,9 +173,6 @@ def run(self):
         except Exception as e:
             self.unexpectedError(e)
 
-        else:
-            self.error('Unknown DomainTools service or invalid data type')
-
 
 if __name__ == '__main__':
     DomainToolsAnalyzer().run()

analyzers/EmlParser/requirements.txt

@@ -1,2 +1,2 @@
 cortexutils;python_version>='3.5'
-git+https://github.com/jeromeleonard/eml_parser.git
+eml_parser

analyzers/FileInfo/FileInfo.json

@@ -1,6 +1,6 @@
 {
   "name": "FileInfo",
-  "version": "3.0",
+  "version": "4.0",
   "author": "TheHive-Project",
   "url": "https://github.com/TheHive-Project/Cortex-Analyzers",
   "license": "AGPL-V3",
@@ -9,6 +9,37 @@
   "baseConfig": "FileInfo",
   "command": "FileInfo/fileinfo_analyzer.py",
   "configurationItems": [
+    {
+      "name": "manalyze_enable",
+      "description": "Wether to enable manalyze submodule or not.",
+      "type": "boolean",
+      "required": true,
+      "multi": false
+    },
+    {
+      "name": "manalyze_enable_docker",
+      "description": "Use docker to run Manalyze.",
+      "type": "boolean",
+      "required": false,
+      "multi": false,
+      "default": false
+    },
+    {
+      "name": "manalyze_enable_binary",
+      "description": "Use local binary to run Manalyze. Need to compile it before!",
+      "type": "boolean",
+      "required": false,
+      "multi": false,
+      "default": true
+    },
+    {
+      "name": "manalyze_binary_path",
+      "description": "Path to the Manalyze binary that was compiled before",
+      "type": "string",
+      "required": false,
+      "multi": false,
+      "default": "/opt/Cortex-Analyzers/utils/manalyze/bin/manalyze"
+    }
   ]
 }
 

analyzers/FileInfo/fileinfo_analyzer.py

@@ -1,10 +1,12 @@
 #!/usr/bin/env python3
 import pyexifinfo
 import magic
+import os
 
 from cortexutils.analyzer import Analyzer
 from submodules import available_submodules
 from submodules.submodule_metadata import MetadataSubmodule
+from submodules.submodule_manalyze import ManalyzeSubmodule
 
 
 class FileInfoAnalyzer(Analyzer):
@@ -15,14 +17,35 @@ def __init__(self):
         self.filetype = pyexifinfo.fileType(self.filepath)
         self.mimetype = magic.Magic(mime=True).from_file(self.filepath)
 
+        # Check if manalyze submodule is enabled
+        if self.get_param('config.manalyze_enable', False, 'Parameter manalyze_enable not given.'
+                                                           'Please enable or disable manalyze submodule explicitly.'):
+            binary_path = self.get_param('config.manalyze_binary_path',
+                                         '/opt/Cortex-Analyzers/utils/manalyze/bin/manalyze')
+            if self.get_param('config.manalyze_enable_docker', False):
+                available_submodules.append(
+                    ManalyzeSubmodule(
+                        use_docker=True
+                    )
+                )
+            elif self.get_param('config.manalyze_enable_binary', False) \
+                    and os.path.isfile(binary_path):
+                available_submodules.append(
+                    ManalyzeSubmodule(
+                        use_binary=True,
+                        binary_path=binary_path
+                    )
+                )
+            else:
+                self.error('Manalyze submodule is enabled, but either there is no method allowed (docker or binary)'
+                           'or the path to binary is not correct.')
+
     def summary(self, raw):
         taxonomies = []
         for submodule in raw['results']:
             taxonomies += submodule['summary']['taxonomies']
         return {'taxonomies': taxonomies}
 
-
-
     def run(self):
         results = []
 
@@ -41,8 +64,8 @@ def run(self):
                 module_results = module.analyze_file(self.filepath)
                 module_summaries = module.module_summary()
                 results.append({
-                   'submodule_name': module.name,
-                   'results': module_results,
+                    'submodule_name': module.name,
+                    'results': module_results,
                     'summary': module_summaries
                 })