Merge pull request #311 from mitre-attack/feature/#310-verify-domains-tactics-techniques-mitigations

Feature/#310 and #314

Author: isaisabel

CHANGELOG.md

@@ -29,6 +29,8 @@
 ### Improvements
 - Added support for deprecated relationships, software and groups. Deprecated relationships, software and groups will not appear on the website UI but can be added to STIX bundles. See issue [#302](https://github.com/mitre-attack/attack-website/issues/302) and [#305](https://github.com/mitre-attack/attack-website/issues/305).
 - Added support for input data with more than one object with the same STIX or ATT&CK ID which can occur if there are multiple versions of the object present in the data. Website will display the most recently modified object depending on the deprecation status. See issue [#304](https://github.com/mitre-attack/attack-website/issues/304).
+- Added domain checks for tactics, software, and mitigations objects. See issue [#310](https://github.com/mitre-attack/attack-website/issues/310).
+- Sorted sub-techniques by ATT&CK ID on Techniques Used tables. See issue [#314](https://github.com/mitre-attack/attack-website/issues/314).
 
 # 20 May 2021
 ## ATT&CK website version 3.2.2

modules/attack_redirections/attack_redirections.py

@@ -30,8 +30,8 @@ def generate_markdown_files(domain):
 
     ms = util.relationshipgetters.get_ms()
 
-    for key in attack_redirections_config.general_redirects_dict:
-        objs = util.stixhelpers.get_all_of_type(ms[domain], key)
+    for types in attack_redirections_config.general_redirects_types_dict:
+        objs = util.stixhelpers.get_all_of_type(ms[domain], types)
         for obj in objs:
             new_attack_id, old_attack_id = get_new_and_old_ids(obj)
 
@@ -44,29 +44,29 @@ def generate_markdown_files(domain):
                         revoked_attack_id = util.buildhelpers.get_attack_id(revoked_by_obj)
 
                         if revoked_attack_id:
-                            generate_obj_redirect(attack_redirections_config.general_redirects_dict[key], revoked_attack_id, old_attack_id, domain)
+                            generate_obj_redirect(attack_redirections_config.general_redirects_dict[types[0]], revoked_attack_id, old_attack_id, domain)
 
                             if old_attack_id != new_attack_id:
-                                generate_obj_redirect(attack_redirections_config.general_redirects_dict[key], revoked_attack_id, new_attack_id, domain)
+                                generate_obj_redirect(attack_redirections_config.general_redirects_dict[types[0]], revoked_attack_id, new_attack_id, domain)
                 else:
-                    generate_obj_redirect(attack_redirections_config.general_redirects_dict[key], new_attack_id, old_attack_id, domain)
+                    generate_obj_redirect(attack_redirections_config.general_redirects_dict[types[0]], new_attack_id, old_attack_id, domain)
 
     if domain == "mobile-attack":
-        for key in attack_redirections_config.mobile_redirect_dict:
-            objs = util.stixhelpers.get_all_of_type(ms[domain], key)
+        for types in attack_redirections_config.mobile_redirect_types_dict:
+            objs = util.stixhelpers.get_all_of_type(ms[domain], types)
             for obj in objs:
                 new_attack_id, old_attack_id = get_new_and_old_ids(obj)
 
                 if new_attack_id:
-                    generate_obj_redirect(attack_redirections_config.mobile_redirect_dict[key], new_attack_id, old_attack_id, domain)
+                    generate_obj_redirect(attack_redirections_config.mobile_redirect_dict[types[0]], new_attack_id, old_attack_id, domain)
 
     generate_tactic_redirects(ms, domain)
 
 
 def generate_tactic_redirects(ms, domain):
     """Responsible for generating tactic redirects markdown"""
 
-    tactics = util.stixhelpers.get_all_of_type(ms[domain], 'x-mitre-tactic')
+    tactics = util.stixhelpers.get_all_of_type(ms[domain], ['x-mitre-tactic'])
 
     data = {}
     for tactic in tactics:

modules/attack_redirections/attack_redirections_config.py

@@ -12,6 +12,12 @@
     "intrusion-set": {"old": "Group", "new": "groups"}
 }
 
+general_redirects_types_dict = [
+    ["attack-pattern"], 
+    ["malware", "tool"], 
+    ["intrusion-set"]
+]
+
 # Mobile redirects
 mobile_redirect_dict = {
     "course-of-action": {
@@ -20,6 +26,10 @@
     }
 }
 
+mobile_redirect_types_dict = [
+    ["course-of-action"]
+]
+
 # File paths dictionary
 redirects_paths = {
     'enterprise-attack': "wiki/", 

modules/matrices/matrices.py

@@ -92,7 +92,7 @@ def generate_deprecated_matrix(matrix, side_menu_data=None):
     # memorystore for the current domain
     domain_ms = ms[matrix['matrix']]
 
-    sub_matrices = util.stixhelpers.get_matrices(domain_ms)
+    sub_matrices = util.stixhelpers.get_matrices(domain_ms, matrix['matrix'])
     data['descr'] = ""
     for sub in sub_matrices:
         if sub.get('description'):
@@ -121,13 +121,13 @@ def get_sub_matrices(matrix):
     # memorystore for the current domain
     domain_ms = ms[matrix['matrix']]
     # get relevant techniques
-    techniques = util.stixhelpers.get_techniques(domain_ms)
+    techniques = util.stixhelpers.get_techniques(domain_ms, matrix['matrix'])
     platform_techniques = util.buildhelpers.filter_techniques_by_platform(techniques, matrix['platforms'])
     platform_techniques = util.buildhelpers.filter_out_subtechniques(platform_techniques)
     # remove revoked
     platform_techniques = util.buildhelpers.filter_deprecated_revoked(platform_techniques)
     # get relevant tactics
-    all_tactics = util.stixhelpers.get_all_of_type(domain_ms, "x-mitre-tactic")
+    all_tactics = util.stixhelpers.get_all_of_type(domain_ms, ["x-mitre-tactic"])
     tactic_id_to_shortname = { tactic["id"]: tactic["x_mitre_shortname"] for tactic in all_tactics }
 
     has_subtechniques = False #track whether the current matrix has subtechniques
@@ -217,7 +217,7 @@ def transform_tactic(tactic_id):
         return obj
 
     data = []
-    sub_matrices = util.stixhelpers.get_matrices(domain_ms)
+    sub_matrices = util.stixhelpers.get_matrices(domain_ms, matrix['matrix'])
     for sub_matrix in sub_matrices:
         # find last modified date
         matrix_dates = util.buildhelpers.get_created_and_modified_dates(sub_matrix)

modules/tactics/tactics.py

@@ -39,8 +39,8 @@ def generate_tactics():
 
     for domain in site_config.domains:
         #Reads the STIX and creates a list of the ATT&CK Techniques
-        techniques_no_sub[domain['name']] = util.buildhelpers.filter_out_subtechniques(util.stixhelpers.get_techniques(ms[domain['name']]))
-        tactics[domain['name']] = util.stixhelpers.get_tactic_list(ms[domain['name']])
+        techniques_no_sub[domain['name']] = util.buildhelpers.filter_out_subtechniques(util.stixhelpers.get_techniques(ms[domain['name']], domain['name']))
+        tactics[domain['name']] = util.stixhelpers.get_tactic_list(ms[domain['name']], domain['name'])
 
     side_nav_data = util.buildhelpers.get_side_nav_domains_data("tactics", tactics)
 

modules/techniques/techniques.py

@@ -43,8 +43,8 @@ def generate_techniques():
 
     for domain in site_config.domains:
         #Reads the STIX and creates a list of the ATT&CK Techniques
-        techniques_no_sub[domain['name']] = util.buildhelpers.filter_out_subtechniques(util.stixhelpers.get_techniques(ms[domain['name']]))
-        tactics[domain['name']] = util.stixhelpers.get_tactic_list(ms[domain['name']])
+        techniques_no_sub[domain['name']] = util.buildhelpers.filter_out_subtechniques(util.stixhelpers.get_techniques(ms[domain['name']], domain['name']))
+        tactics[domain['name']] = util.stixhelpers.get_tactic_list(ms[domain['name']], domain['name'])
 
     side_nav_data = get_technique_side_nav_data(techniques_no_sub, tactics)
 
@@ -488,6 +488,10 @@ def get_technique_side_nav_data(techniques, tactics):
                             child['children'] = []
                             technique_row['children'].append(child)
 
+                    # Sort subtechniques by ATT&CK ID
+                    if technique_row['children']:
+                        technique_row['children'] = sorted(technique_row['children'], key=lambda k: k['id'])
+
                 # Add technique data to tactic
                 tactic_row['children'].append(technique_row)
             # Add tactic to domain

modules/tour/tour.py

@@ -72,7 +72,7 @@ def get_tour_steps(matrix):
         return {}
 
     # Reads the STIX and creates a list of the techniques
-    techniques = util.stixhelpers.get_techniques(ms)
+    techniques = util.stixhelpers.get_techniques(ms, matrix['matrix'])
 
     techs_no_subtechs = util.buildhelpers.filter_out_subtechniques(techniques)
     techs_with_subtechs = util.buildhelpers.filter_out_techniques_without_subtechniques(techniques)
@@ -89,6 +89,10 @@ def get_tour_steps(matrix):
     # Find technique with sub-techniques
     technique = get_technique_with_subtechniques(techs_no_subtechs)
 
+    if not technique:
+        # Did not find technique with sub-technique
+        return steps
+
     # Get technique ID and store that as the step
     steps['technique'] = "techniques/{}".format(util.buildhelpers.get_attack_id(technique))
     subtechnique_attack_id = get_subtech_n_of_technique(technique)
@@ -178,6 +182,7 @@ def get_technique_with_subtechniques(techs_no_subtechs):
         Return technique with the most sub-techniques if not the case 
     """
 
+    subtech_count_min = 4
     counter = 0
     chosen_tech = {}
 
@@ -191,7 +196,7 @@ def get_technique_with_subtechniques(techs_no_subtechs):
             # Check if sub-technique count is bigger than counter
             if counter < subtech_count:
                 # Quick return if found
-                if subtech_count > 3: return tech
+                if subtech_count >= subtech_count_min: return tech
                 # Set counter and new techique
                 counter = subtech_count
                 chosen_tech = tech

modules/util/buildhelpers.py

@@ -509,6 +509,8 @@ def technique_used_helper(technique_list, technique, reference_list):
                     technique_list[parent_id] = parent_technique_used_helper(parent_id)
 
                 technique_list[parent_id]['subtechniques'].append(get_technique_data_helper(attack_id, technique, reference_list))
+                # Sort subtechniques by name
+                technique_list[parent_id]['subtechniques'] = sorted(technique_list[parent_id]['subtechniques'], key=lambda k: k['id'])
 
             # Attack id is regular technique
             else:

modules/util/stixhelpers.py

@@ -20,49 +20,72 @@ def get_mitigation_list(src):
 
     return sorted(mitigations, key=lambda k: k['name'].lower())
 
-def get_matrices(src):
+def get_matrices(src, domain):
     """Reads the STIX and returns a list of all matrices in the STIX"""
 
     matrices = src.query([
         stix2.Filter('type', '=', 'x-mitre-matrix'),
     ])
 
+    # Filter out by domain
+    matrices = [x for x in matrices if not hasattr(x, 'x_mitre_domains') or domain in x.get('x_mitre_domains')]
+
     return matrices
 
-def get_tactic_list(src, matrix_id=None):
+def get_tactic_list(src, domain, matrix_id=None):
     """Reads the STIX and returns a list of all tactics in the STIX"""
 
     tactics = []
-    matrix = src.query([
+    matrices = src.query([
         stix2.Filter('type', '=', 'x-mitre-matrix'),
     ])
 
+    matrices = sorted(matrices, key=lambda k: len(k['tactic_refs']), reverse=True)
+
     if matrix_id:
-        for curr_matrix in matrix:
+        for curr_matrix in matrices:
             if curr_matrix['id'] == matrix_id:
                 for tactic_id in curr_matrix['tactic_refs']:
                     tactics.append(src.query([stix2.Filter('id', '=', tactic_id)])[0])    
     else:
-        for i in range(len(matrix)):
-            for tactic_id in matrix[i]['tactic_refs']:
-                tactics.append(src.query([stix2.Filter('id', '=', tactic_id)])[0])    
+        for matrix in matrices:
+            for tactic_id in matrix['tactic_refs']:
+                tactics.append(src.query([stix2.Filter('id', '=', tactic_id)])[0])
+    
+    # Filter out by domain
+    tactics = [x for x in tactics if not hasattr(x, 'x_mitre_domains') or domain in x.get('x_mitre_domains')]
 
     return tactics
 
-def get_all_of_type(src, obj_type):
+def get_all_of_type(src, types):
     """Reads the STIX and returns a list of all of a particular
-       type of object in the STIX
+       type of object in the STIX, removes duplicate STIX and ATT&CK IDs 
     """
 
-    return src.query([stix2.Filter('type', '=', obj_type)])
+    def grab_filtered_list_by_type(stix_type):
+        return src.query([stix2.Filter('type', '=', stix_type)])
+
+    stix_objs = {}
+    attack_id_objs = {}
 
-def get_techniques(src):
-    """Reads the STIX and returns a list of all techniques in the STIX"""
+    for stix_type in types:
+        result = grab_filtered_list_by_type(stix_type)
+        for obj in result:
+            add_replace_or_ignore(stix_objs, attack_id_objs, obj)
+
+    return [attack_id_objs[key] for key in attack_id_objs]
+
+def get_techniques(src, domain):
+    """Reads the STIX and returns a list of all techniques in the STIX
+       by given domain
+    """
 
     tech_list = src.query([
         stix2.Filter('type', '=', 'attack-pattern'),
         stix2.Filter('revoked', '=', False)
     ])
+    # Filter out by domain
+    tech_list = [x for x in tech_list if not hasattr(x, 'x_mitre_domains') or domain in x.get('x_mitre_domains')]
 
     tech_list = sorted(tech_list, key=lambda k: k['name'].lower())
     return tech_list
@@ -130,8 +153,14 @@ def get_technique_id_domain_map(ms):
         for val in curr_list:
             technique_id = buildhelpers.get_attack_id(val)
             if technique_id:
-                tech_list[technique_id] = domain['name']
-    
+                if val.get('x_mitre_domains'):
+                    if domain['name'] in val['x_mitre_domains']:
+                        if tech_list.get(technique_id) and domain['name'] not in tech_list[technique_id]:
+                            tech_list[technique_id].append(domain['name'])
+                        else:
+                            tech_list[technique_id] = domain['name']
+                else:
+                    tech_list[technique_id] = domain['name']
     return tech_list
 
 def add_replace_or_ignore(stix_objs, attack_id_objs, obj_in_question):
@@ -144,7 +173,7 @@ def add_replace_or_ignore(stix_objs, attack_id_objs, obj_in_question):
 
     def has_STIX_ATTACK_ID_conflict(attack_id):
         # Check if STIX ID has been seen before, if it has, return ATT&CK ID of conflict ATT&CK if ATT&CK IDs are different
-        conflict = stix_objs.get(obj_in_question.id)
+        conflict = stix_objs.get(obj_in_question.get('id'))
         if conflict:
             conflict_attack_id = buildhelpers.get_attack_id(conflict)
             if conflict_attack_id != attack_id and attack_id_objs.get(conflict_attack_id):
@@ -162,7 +191,7 @@ def replace_object(attack_id, conflict_attack_id):
         else:
             attack_id_objs[attack_id] = obj_in_question
 
-        stix_objs[obj_in_question.id] = obj_in_question
+        stix_objs[obj_in_question.get('id')] = obj_in_question
 
     # Get ATT&CK ID
     attack_id = buildhelpers.get_attack_id(obj_in_question)
@@ -176,7 +205,7 @@ def replace_object(attack_id, conflict_attack_id):
 
     # Check if object in conflict exists
     # STIX ID
-    stix_id_obj_in_conflict = stix_objs.get(obj_in_question.id)
+    stix_id_obj_in_conflict = stix_objs.get(obj_in_question.get('id'))
 
     # Get ATT&CK ID conflicts
     if conflict_attack_id:
@@ -189,7 +218,7 @@ def replace_object(attack_id, conflict_attack_id):
 
     if not stix_id_obj_in_conflict:
         # Add if object does not exist in STIX ID map
-        stix_objs[obj_in_question.id] = obj_in_question
+        stix_objs[obj_in_question.get('id')] = obj_in_question
 
     if not attack_id_obj_in_conflict:
         # Add if object does not exist in ATT&CK ID map