From f19c10f9c3cf6bf04f283c50a99c880fd0e82ebf Mon Sep 17 00:00:00 2001 From: kx499 Date: Mon, 18 Feb 2019 03:16:58 +0000 Subject: [PATCH 1/4] Updated to add in Cert SHA1 for pivots --- analyzers/Crtsh/crtshquery.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/analyzers/Crtsh/crtshquery.py b/analyzers/Crtsh/crtshquery.py index c3dd6e3c0..dc0585171 100755 --- a/analyzers/Crtsh/crtshquery.py +++ b/analyzers/Crtsh/crtshquery.py @@ -3,6 +3,7 @@ import requests import json +import re from cortexutils.analyzer import Analyzer @@ -29,6 +30,7 @@ def search(self, domain, wildcard=True): XML notation would also include the base64 cert: https://crt.sh/atom?q={} """ + rex = '\SHA-1\(Certificate\)\\s+\([^\<]+)\' base_url = "https://crt.sh/?q={}&output=json" if wildcard: domain = "%25.{}".format(domain) @@ -41,9 +43,18 @@ def search(self, domain, wildcard=True): try: content = req.content.decode('utf-8') data = json.loads(content.replace('}{', '},{')) + for c in data: + det_url = 'https://crt.sh/?q={}&output=json'.format(c['min_cert_id']) + det_req = requests.get(det_url, headers={'User-Agent': ua}) + if det_req.status_code == requests.codes.ok: + det_con = det_req.content.decode('utf-8') + sha1 = re.findall(rex, det_con)[0] + c['sha1'] = sha1 + else: + c['sha1'] = '' return data - except Exception: - self.error("Error retrieving information.") + except Exception as e: + self.error("Error retrieving information. {}".format(e)) return None def __init__(self): From ac31aba538d7532a92aefe4dc321ad43acc651c4 Mon Sep 17 00:00:00 2001 From: kx499 Date: Mon, 18 Feb 2019 03:28:57 +0000 Subject: [PATCH 2/4] updated log report for sha1 and changed from table layout --- .../Crt_sh_Transparency_Logs_1_0/long.html | 34 +++++++------------ 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/thehive-templates/Crt_sh_Transparency_Logs_1_0/long.html b/thehive-templates/Crt_sh_Transparency_Logs_1_0/long.html index d2694997f..319a0b3ea 100644 --- a/thehive-templates/Crt_sh_Transparency_Logs_1_0/long.html +++ b/thehive-templates/Crt_sh_Transparency_Logs_1_0/long.html @@ -7,28 +7,18 @@

No result found.

- - - - - - - - - - - - - - - - - - - - - -
namenot beforenot aftermin cert idissuer nameissuer ca idmin entry timestamp
{{r.name_value}}{{r.not_before}}{{r.not_after}}{{r.min_cert_id}}{{r.issuer_name}}{{r.issuer_ca_id}}{{r.min_entry_timestamp}}
+
+
Name:
+
{{r.name_value}}
+
SHA1:
+
{{r.sha1}}
+
Issuer:
+
{{r.issuer_name}}
+
Not Before:
+
{{r.not_before}}
+
Not After:
+
{{r.not_after}}
+
From 7b69cfb63787dd5129f9d67192e7b5a3d87bafef Mon Sep 17 00:00:00 2001 From: tssbo82 Date: Tue, 19 Feb 2019 16:01:23 -0500 Subject: [PATCH 3/4] update to the crtsh analyzer to fix my errors --- analyzers/Crtsh/crtshquery.py | 44 +++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/analyzers/Crtsh/crtshquery.py b/analyzers/Crtsh/crtshquery.py index dc0585171..14d6c93be 100755 --- a/analyzers/Crtsh/crtshquery.py +++ b/analyzers/Crtsh/crtshquery.py @@ -32,30 +32,44 @@ def search(self, domain, wildcard=True): """ rex = '\SHA-1\(Certificate\)\\s+\([^\<]+)\' base_url = "https://crt.sh/?q={}&output=json" - if wildcard: - domain = "%25.{}".format(domain) url = base_url.format(domain) - ua = 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1' + ua = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36' req = requests.get(url, headers={'User-Agent': ua}) if req.ok: try: content = req.content.decode('utf-8') data = json.loads(content.replace('}{', '},{')) - for c in data: - det_url = 'https://crt.sh/?q={}&output=json'.format(c['min_cert_id']) - det_req = requests.get(det_url, headers={'User-Agent': ua}) - if det_req.status_code == requests.codes.ok: - det_con = det_req.content.decode('utf-8') - sha1 = re.findall(rex, det_con)[0] - c['sha1'] = sha1 - else: - c['sha1'] = '' - return data except Exception as e: - self.error("Error retrieving information. {}".format(e)) - return None + self.error("Error retrieving base domain information. {}".format(e)) + return None + + if wildcard: + url2 = base_url.format("%25{}.".format(domain)) + req2 = requests.get(url2, headers={'User-Agent': ua}) + if req2.ok: + try: + content2 = req2.content.decode('utf-8') + data2 = json.loads(content2.replace('}{', '},{')) + data.extend(data2) + except Exception as e: + self.error("Error retrieving wildcard information. {}".format(e)) + return None + + for c in data: + det_url = 'https://crt.sh/?q={}&output=json'.format(c['min_cert_id']) + try: + det_req = requests.get(det_url, headers={'User-Agent': ua}) + if det_req.status_code == requests.codes.ok: + det_con = det_req.content.decode('utf-8') + sha1 = re.findall(rex, det_con)[0] + c['sha1'] = sha1 + else: + c['sha1'] = '' + except: + c['sha1'] = '' + return data def __init__(self): Analyzer.__init__(self) From 7432eacef18bdb0aee346019875b580b4c3eeca1 Mon Sep 17 00:00:00 2001 From: Faber Date: Fri, 22 Feb 2019 14:41:11 -0500 Subject: [PATCH 4/4] last update to crt.sh - added manual artifact parsing --- analyzers/Crtsh/crtshquery.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/analyzers/Crtsh/crtshquery.py b/analyzers/Crtsh/crtshquery.py index 14d6c93be..6667b753b 100755 --- a/analyzers/Crtsh/crtshquery.py +++ b/analyzers/Crtsh/crtshquery.py @@ -93,6 +93,19 @@ def summary(self, raw): return {"taxonomies": taxonomies} + def artifacts(self, raw): + artifacts = [] + results = raw.get('certobj', {}).get('result', []) + for cert in results: + if 'sha1' in cert: + artifacts.append({'type':'certificate_hash', 'value':cert.get('sha1')}) + if 'name_value' in cert: + artifacts.append({'type': 'fqdn', 'value': cert.get('name_value')}) + + #dedup + artifacts = [dict(t) for t in {tuple(d.items()) for d in artifacts}] + return artifacts + def run(self): Analyzer.run(self)