From 8ef5f40ef4a089027dfb68d4b15b4705e3885377 Mon Sep 17 00:00:00 2001 From: Davide Arcuri Date: Thu, 5 Mar 2020 14:17:59 +0100 Subject: [PATCH] Added url scan feature --- analyzers/Urlscan.io/Urlscan_Scan.json | 23 ++++ analyzers/Urlscan.io/Urlscan_Search.json | 15 ++- analyzers/Urlscan.io/urlscan.py | 33 +++++ analyzers/Urlscan.io/urlscan_analyzer.py | 93 ++++++++++---- .../Urlscan_io_Scan_0_1_0/long.html | 117 ++++++++++++++++++ .../short.html | 0 .../long.html | 0 .../Urlscan_io_Search_0_1_1/short.html | 3 + 8 files changed, 253 insertions(+), 31 deletions(-) create mode 100644 analyzers/Urlscan.io/Urlscan_Scan.json create mode 100644 thehive-templates/Urlscan_io_Scan_0_1_0/long.html rename thehive-templates/{Urlscan_io_Search_0_1_0 => Urlscan_io_Scan_0_1_0}/short.html (100%) rename thehive-templates/{Urlscan_io_Search_0_1_0 => Urlscan_io_Search_0_1_1}/long.html (100%) create mode 100644 thehive-templates/Urlscan_io_Search_0_1_1/short.html diff --git a/analyzers/Urlscan.io/Urlscan_Scan.json b/analyzers/Urlscan.io/Urlscan_Scan.json new file mode 100644 index 000000000..f34ce9af1 --- /dev/null +++ b/analyzers/Urlscan.io/Urlscan_Scan.json @@ -0,0 +1,23 @@ +{ + "name": "Urlscan.io_Scan", + "author": "ninoseki, Kyle Parrish (@arnydo)", + "license": "MIT", + "url": "https://github.com/arnydo/Cortex-Analyzers", + "version": "0.1.0", + "description": "Scan URLs on urlscan.io", + "dataTypeList": ["url", "domain", "fqdn"], + "command": "Urlscan.io/urlscan_analyzer.py", + "baseConfig": "Urlscan.io", + "config": { + "service":"scan" + }, + "configurationItems": [ + { + "name": "key", + "description": "API key for Urlscan.io", + "type": "string", + "multi": false, + "required": true + } + ] +} diff --git a/analyzers/Urlscan.io/Urlscan_Search.json b/analyzers/Urlscan.io/Urlscan_Search.json index eb5448a78..a8984ea02 100644 --- a/analyzers/Urlscan.io/Urlscan_Search.json +++ b/analyzers/Urlscan.io/Urlscan_Search.json @@ -1,11 +1,14 @@ { "name": "Urlscan.io_Search", - "author": "ninoseki", + "author": "ninoseki, Kyle Parrish (@arnydo)", "license": "MIT", - "url": "https://github.com/ninoseki/cortex_urlscan_analyzer", - "version": "0.1.0", - "baseConfig": "Urlscan", + "url": "https://github.com/arnydo/Cortex-Analyzers", + "version": "0.1.1", "description": "Search IPs, domains, hashes or URLs on urlscan.io", - "dataTypeList": ["ip", "domain", "hash", "url"], - "command": "Urlscan.io/urlscan_analyzer.py" + "dataTypeList": ["ip", "domain", "hash", "fqdn", "url"], + "command": "Urlscan.io/urlscan_analyzer.py", + "baseConfig": "Urlscan.io", + "config": { + "service": "get" + } } diff --git a/analyzers/Urlscan.io/urlscan.py b/analyzers/Urlscan.io/urlscan.py index 3c364bc54..dd084122e 100755 --- a/analyzers/Urlscan.io/urlscan.py +++ b/analyzers/Urlscan.io/urlscan.py @@ -1,5 +1,6 @@ import requests import json +import time class UrlscanException(Exception): @@ -18,3 +19,35 @@ def search(self): return r.json() else: raise UrlscanException("urlscan.io returns %s" % r.status_code) + + def scan(self, api_key): + headers = { + "Content-Type": "application/json", + "API-Key": api_key, + } + data = '{"url": %s, "public": "on"}' % self.query + r = requests.post( + "https://urlscan.io/api/v1/scan/", headers=headers, data=data, verify=False + ) + if r.status_code == 200: + submission_url = r.json()["api"] + + finished = False + tries = 0 + while tries <= 15: + submission_req = requests.get(submission_url) + if submission_req.status_code == 200: + return submission_req.json() + tries += 1 + time.sleep(20) + + raise UrlscanException( + "urlscan.io returns {0} and data was {1} on url {2}".format( + submission_req.status_code, data, submission_url + ) + ) + + else: + raise UrlscanException( + "urlscan.io returns {0} and data was {1}".format(r.status_code, data) + ) diff --git a/analyzers/Urlscan.io/urlscan_analyzer.py b/analyzers/Urlscan.io/urlscan_analyzer.py index a43f2596b..7846e1ac9 100755 --- a/analyzers/Urlscan.io/urlscan_analyzer.py +++ b/analyzers/Urlscan.io/urlscan_analyzer.py @@ -6,6 +6,9 @@ class UrlscanAnalyzer(Analyzer): def __init__(self): Analyzer.__init__(self) + self.service = self.get_param('config.service', None, 'Service parameter is missing') + if self.service == 'scan': + self.api_key = self.get_param('config.key', None, 'Missing URLScan API key') def search(self, indicator): """ @@ -17,40 +20,80 @@ def search(self, indicator): res = Urlscan(indicator).search() return res + def scan(self, indicator): + """ + Scans a website for indicators + :param indicator: url + :type indicator: str + :return: dict + """ + res = Urlscan(indicator).scan(self.api_key) + return res + def run(self): - targets = ['ip', 'domain', 'hash', 'url'] - if self.data_type == 'url': - query = '"{}"'.format(self.get_data()) + if self.service == 'scan': + if self.data_type in ['domain', 'url', 'fqdn']: + query = '"{}"'.format(self.get_data()) + try: + self.report({ + 'type': self.data_type, + 'query': query, + 'service': self.service, + 'indicator': self.scan(query) + }) + except UrlscanException as err: + self.error(str(err)) + else: + self.error('Invalid data type. URL expected') + elif self.service == 'get': + targets = ['ip', 'domain', 'fqdn', 'hash', 'url'] + if self.data_type == 'url': + query = '"{}"'.format(self.get_data()) + else: + query = self.get_data() + + try: + if self.data_type in targets: + self.report({ + 'type': self.data_type, + 'query': query, + 'service': self.service, + 'indicator': self.search(query) + }) + except UrlscanException as err: + self.error(str(err)) else: - query = self.get_data() - - try: - if self.data_type in targets: - self.report({ - 'type': self.data_type, - 'query': query, - 'indicator': self.search(query) - }) - except UrlscanException as err: - self.error(str(err)) + self.error('Invalid service') + def summary(self, raw): taxonomies = [] level = "info" namespace = "urlscan.io" - predicate = "Search" - - total = raw["indicator"]["total"] - if total <= 1: - level = 'suspicious' if total == 1 else 'info' - value = "{} result".format(total) - taxonomies.append(self.build_taxonomy( - level, namespace, predicate, value)) + predicate = "Search" if raw["service"] == 'get' else "Scan" + + if predicate == "Search": + total = raw["indicator"]["total"] + if total <= 1: + level = 'suspicious' if total == 1 else 'info' + value = "{} result".format(total) + taxonomies.append(self.build_taxonomy( + level, namespace, predicate, value)) + else: + level = 'suspicious' + value = "{} results".format(total) + taxonomies.append(self.build_taxonomy( + level, namespace, predicate, value)) else: - level = 'suspicious' - value = "{} results".format(total) + score = raw["indicator"]["verdicts"]["overall"]["score"] + value = "Overall Score:{}".format(score) + malicious = raw["indicator"]["verdicts"]["overall"]["malicious"] + if malicious: + level = 'malicious' + elif score > 0: + level = 'suspicious' taxonomies.append(self.build_taxonomy( - level, namespace, predicate, value)) + level, namespace, predicate, value)) return {"taxonomies": taxonomies} diff --git a/thehive-templates/Urlscan_io_Scan_0_1_0/long.html b/thehive-templates/Urlscan_io_Scan_0_1_0/long.html new file mode 100644 index 000000000..b1ae37973 --- /dev/null +++ b/thehive-templates/Urlscan_io_Scan_0_1_0/long.html @@ -0,0 +1,117 @@ +
+
+ Urlscan.io Scan - General info +
+
+
+
Url
+
{{content.indicator.page.url}}
+
+
+
Domain
+
{{content.indicator.page.domain}}
+
+
+
Report
+
+ + {{content.indicator.task.reportURL}} + +
+
+
+
Ip
+
{{content.indicator.page.ip}}
+
+
+
Country
+
{{content.indicator.page.country}} - {{content.indicator.page.city}}
+
+
+
+ +
+
+ Verdicts +
+
+
+
Community score:
+
+ {{content.indicator.verdicts.community.score}} + [Malicious votes: {{content.indicator.verdicts.community.votesMalicious}}/{{content.indicator.verdicts.community.votesTotal}}] +
+
+
+
Engines score:
+
+ {{content.indicator.verdicts.engines.score}} + [Malicious engines: {{content.indicator.verdicts.engines.maliciousTotal}}/{{content.indicator.verdicts.engines.enginesTotal}}] +
+
+
+
Urlscan score:
+
+ {{content.indicator.verdicts.urlscan.score}} + + [malicious] + +
+
+
+
Overall score:
+
+ {{content.indicator.verdicts.overall.score}} + + [malicious] + +
+
+
+
+ +
+
+ Items found +
+
+ + + + + + + + + + + + + + + + + + + + + + + +
IPasnsCountriesHashesServersUrlsDomains
{{content.indicator.lists.ips.length}}{{content.indicator.lists.asns.length}}{{content.indicator.lists.countries.length}}{{content.indicator.lists.hashes.length}}{{content.indicator.lists.servers.length}}{{content.indicator.lists.urls.length}}{{content.indicator.lists.domains.length}}
+
+
+ + +
+
+ {{artifact.data | fang}} +
+
+
+
+ urlscan.io:
+
{{content.errorMessage}}
+
+
+
diff --git a/thehive-templates/Urlscan_io_Search_0_1_0/short.html b/thehive-templates/Urlscan_io_Scan_0_1_0/short.html similarity index 100% rename from thehive-templates/Urlscan_io_Search_0_1_0/short.html rename to thehive-templates/Urlscan_io_Scan_0_1_0/short.html diff --git a/thehive-templates/Urlscan_io_Search_0_1_0/long.html b/thehive-templates/Urlscan_io_Search_0_1_1/long.html similarity index 100% rename from thehive-templates/Urlscan_io_Search_0_1_0/long.html rename to thehive-templates/Urlscan_io_Search_0_1_1/long.html diff --git a/thehive-templates/Urlscan_io_Search_0_1_1/short.html b/thehive-templates/Urlscan_io_Search_0_1_1/short.html new file mode 100644 index 000000000..5fc0dabfb --- /dev/null +++ b/thehive-templates/Urlscan_io_Search_0_1_1/short.html @@ -0,0 +1,3 @@ + + {{t.namespace}}:{{t.predicate}}="{{t.value}}" +