From a861ce4d41c17bc8a56815241dd7149019f8ab41 Mon Sep 17 00:00:00 2001 From: Jonhnathan <26856693+w0rk3r@users.noreply.github.com> Date: Wed, 6 Sep 2023 18:30:05 -0300 Subject: [PATCH 1/9] [Security Content] Introduce Investigate Plugin in Investigation Guides --- ...d_control_certutil_network_connection.toml | 30 +++++++++++++++++-- ...ommand_and_control_common_webservices.toml | 30 +++++++++++++++++-- ...ol_remote_file_copy_desktopimgdownldr.toml | 27 ++++++++++++++++- ...and_control_remote_file_copy_mpcmdrun.toml | 27 ++++++++++++++++- ...d_control_remote_file_copy_powershell.toml | 27 ++++++++++++++++- 5 files changed, 132 insertions(+), 9 deletions(-) diff --git a/rules/windows/command_and_control_certutil_network_connection.toml b/rules/windows/command_and_control_certutil_network_connection.toml index 2d83c3f1df0..4ebc7ff6931 100644 --- a/rules/windows/command_and_control_certutil_network_connection.toml +++ b/rules/windows/command_and_control_certutil_network_connection.toml @@ -4,7 +4,7 @@ integration = ["endpoint", "windows"] maturity = "production" min_stack_comments = "New fields added: required_fields, related_integrations, setup" min_stack_version = "8.3.0" -updated_date = "2023/06/22" +updated_date = "2023/09/06" [transform] [[transform.osquery]] @@ -31,6 +31,28 @@ services.path FROM services JOIN authenticode ON services.path = authenticode.pa authenticode.path JOIN hash ON services.path = hash.path WHERE authenticode.result != 'trusted' """ +[[transform.investigate]] +label = "Alerts associated with the user in the last 48h" +providers = """ +[[{"field":"event.kind","excluded":false,"queryType":"phrase","value":"signal","valueType":"string"},{"field":"user.id","excluded":false,"queryType":"phrase","value":"{{user.id}}","valueType":"string"}]] +""" +relativeFrom = "now-48h/h" +relativeTo = "now" + +[[transform.investigate]] +label = "Alerts associated with the host in the last 48h" +providers = """ +[[{"field":"event.kind","excluded":false,"queryType":"phrase","value":"signal","valueType":"string"},{"field":"host.name","excluded":false,"queryType":"phrase","value":"{{host.name}}","valueType":"string"}]] +""" +relativeFrom = "now-48h/h" +relativeTo = "now" + +[[transform.investigate]] +label = "Investigate the Subject Process Network Events" +providers = """ +[[{"field":"process.entity_id","excluded":false,"queryType":"phrase","value":"{{process.entity_id}}","valueType":"string"},{"field":"event.category","excluded":false,"queryType":"phrase","value":"network","valueType":"string"}]] +""" + [rule] author = ["Elastic"] @@ -56,8 +78,10 @@ This rule looks for network events where `certutil.exe` contacts IP ranges other #### Possible investigation steps -- Investigate the script execution chain (parent process tree) for unknown processes. Examine their executable files for prevalence, whether they are located in expected locations, and if they are signed with valid digital signatures. +- Investigate the process execution chain (parent process tree) for unknown processes. Examine their executable files for prevalence, whether they are located in expected locations, and if they are signed with valid digital signatures. - Investigate other alerts associated with the user/host during the past 48 hours. + - $investigate_0 + - $investigate_1 - Investigate if the downloaded file was executed. - Determine the context in which `certutil.exe` and the file were run. - Examine the host for derived artifacts that indicate suspicious activities: @@ -65,6 +89,7 @@ This rule looks for network events where `certutil.exe` contacts IP ranges other - Observe and collect information about the following activities in both the sandbox and the alert subject host: - Attempts to contact external domains and addresses. - Use the Elastic Defend network events to determine domains and addresses contacted by the subject process by filtering by the process' `process.entity_id`. + - $investigate_2 - Examine the DNS cache for suspicious or anomalous entries. - $osquery_0 - Use the Elastic Defend registry events to examine registry keys accessed, modified, or created by the related processes in the process tree. @@ -73,7 +98,6 @@ This rule looks for network events where `certutil.exe` contacts IP ranges other - $osquery_2 - $osquery_3 - Retrieve the files' SHA-256 hash values using the PowerShell `Get-FileHash` cmdlet and search for the existence and reputation of the hashes in resources like VirusTotal, Hybrid-Analysis, CISCO Talos, Any.run, etc. -- Investigate potentially compromised accounts. Analysts can do this by searching for login events (for example, 4624) to the target host after the registry modification. ### False positive analysis diff --git a/rules/windows/command_and_control_common_webservices.toml b/rules/windows/command_and_control_common_webservices.toml index 1022834c103..b188ec45c41 100644 --- a/rules/windows/command_and_control_common_webservices.toml +++ b/rules/windows/command_and_control_common_webservices.toml @@ -4,7 +4,7 @@ integration = ["endpoint"] maturity = "production" min_stack_comments = "New fields added: required_fields, related_integrations, setup" min_stack_version = "8.3.0" -updated_date = "2023/06/22" +updated_date = "2023/09/06" [transform] [[transform.osquery]] @@ -31,6 +31,28 @@ services.path FROM services JOIN authenticode ON services.path = authenticode.pa authenticode.path JOIN hash ON services.path = hash.path WHERE authenticode.result != 'trusted' """ +[[transform.investigate]] +label = "Alerts associated with the user in the last 48h" +providers = """ +[[{"field":"event.kind","excluded":false,"queryType":"phrase","value":"signal","valueType":"string"},{"field":"user.id","excluded":false,"queryType":"phrase","value":"{{user.id}}","valueType":"string"}]] +""" +relativeFrom = "now-48h/h" +relativeTo = "now" + +[[transform.investigate]] +label = "Alerts associated with the host in the last 48h" +providers = """ +[[{"field":"event.kind","excluded":false,"queryType":"phrase","value":"signal","valueType":"string"},{"field":"host.name","excluded":false,"queryType":"phrase","value":"{{host.name}}","valueType":"string"}]] +""" +relativeFrom = "now-48h/h" +relativeTo = "now" + +[[transform.investigate]] +label = "Investigate the Subject Process Network Events" +providers = """ +[[{"field":"process.entity_id","excluded":false,"queryType":"phrase","value":"{{process.entity_id}}","valueType":"string"},{"field":"event.category","excluded":false,"queryType":"phrase","value":"network","valueType":"string"}]] +""" + [rule] author = ["Elastic"] @@ -58,8 +80,10 @@ This rule looks for processes outside known legitimate program locations communi #### Possible investigation steps -- Investigate the script execution chain (parent process tree) for unknown processes. Examine their executable files for prevalence, whether they are located in expected locations, and if they are signed with valid digital signatures. +- Investigate the process execution chain (parent process tree) for unknown processes. Examine their executable files for prevalence, whether they are located in expected locations, and if they are signed with valid digital signatures. - Investigate other alerts associated with the user/host during the past 48 hours. + - $investigate_0 + - $investigate_1 - Verify whether the digital signature exists in the executable. - Identify the operation type (upload, download, tunneling, etc.). - Examine the host for derived artifacts that indicate suspicious activities: @@ -67,6 +91,7 @@ This rule looks for processes outside known legitimate program locations communi - Observe and collect information about the following activities in both the sandbox and the alert subject host: - Attempts to contact external domains and addresses. - Use the Elastic Defend network events to determine domains and addresses contacted by the subject process by filtering by the process' `process.entity_id`. + - $investigate_2 - Examine the DNS cache for suspicious or anomalous entries. - $osquery_0 - Use the Elastic Defend registry events to examine registry keys accessed, modified, or created by the related processes in the process tree. @@ -75,7 +100,6 @@ This rule looks for processes outside known legitimate program locations communi - $osquery_2 - $osquery_3 - Retrieve the files' SHA-256 hash values using the PowerShell `Get-FileHash` cmdlet and search for the existence and reputation of the hashes in resources like VirusTotal, Hybrid-Analysis, CISCO Talos, Any.run, etc. -- Investigate potentially compromised accounts. Analysts can do this by searching for login events (for example, 4624) to the target host after the registry modification. ### False positive analysis diff --git a/rules/windows/command_and_control_remote_file_copy_desktopimgdownldr.toml b/rules/windows/command_and_control_remote_file_copy_desktopimgdownldr.toml index 4b869245b62..084da1f0178 100644 --- a/rules/windows/command_and_control_remote_file_copy_desktopimgdownldr.toml +++ b/rules/windows/command_and_control_remote_file_copy_desktopimgdownldr.toml @@ -2,7 +2,7 @@ creation_date = "2020/09/03" integration = ["endpoint", "windows"] maturity = "production" -updated_date = "2023/06/22" +updated_date = "2023/09/06" min_stack_comments = "New fields added: required_fields, related_integrations, setup" min_stack_version = "8.3.0" @@ -31,6 +31,28 @@ services.path FROM services JOIN authenticode ON services.path = authenticode.pa authenticode.path JOIN hash ON services.path = hash.path WHERE authenticode.result != 'trusted' """ +[[transform.investigate]] +label = "Alerts associated with the user in the last 48h" +providers = """ +[[{"field":"event.kind","excluded":false,"queryType":"phrase","value":"signal","valueType":"string"},{"field":"user.id","excluded":false,"queryType":"phrase","value":"{{user.id}}","valueType":"string"}]] +""" +relativeFrom = "now-48h/h" +relativeTo = "now" + +[[transform.investigate]] +label = "Alerts associated with the host in the last 48h" +providers = """ +[[{"field":"event.kind","excluded":false,"queryType":"phrase","value":"signal","valueType":"string"},{"field":"host.name","excluded":false,"queryType":"phrase","value":"{{host.name}}","valueType":"string"}]] +""" +relativeFrom = "now-48h/h" +relativeTo = "now" + +[[transform.investigate]] +label = "Investigate the Subject Process Network Events" +providers = """ +[[{"field":"process.entity_id","excluded":false,"queryType":"phrase","value":"{{process.entity_id}}","valueType":"string"},{"field":"event.category","excluded":false,"queryType":"phrase","value":"network","valueType":"string"}]] +""" + [rule] author = ["Elastic"] @@ -60,6 +82,8 @@ The `Desktopimgdownldr.exe` utility is used to to configure lockscreen/desktop i - Identify the user account that performed the action and whether it should perform this kind of action. - Contact the account owner and confirm whether they are aware of this activity. - Investigate other alerts associated with the user/host during the past 48 hours. + - $investigate_0 + - $investigate_1 - Assess whether this behavior is prevalent in the environment by looking for similar occurrences across hosts. - Check the reputation of the domain or IP address used to host the downloaded file or if the user downloaded the file from an internal system. - Examine the host for derived artifacts that indicate suspicious activities: @@ -67,6 +91,7 @@ The `Desktopimgdownldr.exe` utility is used to to configure lockscreen/desktop i - Observe and collect information about the following activities in both the sandbox and the alert subject host: - Attempts to contact external domains and addresses. - Use the Elastic Defend network events to determine domains and addresses contacted by the subject process by filtering by the process' `process.entity_id`. + - $investigate_2 - Examine the DNS cache for suspicious or anomalous entries. - $osquery_0 - Use the Elastic Defend registry events to examine registry keys accessed, modified, or created by the related processes in the process tree. diff --git a/rules/windows/command_and_control_remote_file_copy_mpcmdrun.toml b/rules/windows/command_and_control_remote_file_copy_mpcmdrun.toml index 990ecb64211..28255cc86f7 100644 --- a/rules/windows/command_and_control_remote_file_copy_mpcmdrun.toml +++ b/rules/windows/command_and_control_remote_file_copy_mpcmdrun.toml @@ -2,7 +2,7 @@ creation_date = "2020/09/03" integration = ["endpoint", "windows"] maturity = "production" -updated_date = "2023/06/22" +updated_date = "2023/09/06" min_stack_comments = "New fields added: required_fields, related_integrations, setup" min_stack_version = "8.3.0" @@ -31,6 +31,28 @@ services.path FROM services JOIN authenticode ON services.path = authenticode.pa authenticode.path JOIN hash ON services.path = hash.path WHERE authenticode.result != 'trusted' """ +[[transform.investigate]] +label = "Alerts associated with the user in the last 48h" +providers = """ +[[{"field":"event.kind","excluded":false,"queryType":"phrase","value":"signal","valueType":"string"},{"field":"user.id","excluded":false,"queryType":"phrase","value":"{{user.id}}","valueType":"string"}]] +""" +relativeFrom = "now-48h/h" +relativeTo = "now" + +[[transform.investigate]] +label = "Alerts associated with the host in the last 48h" +providers = """ +[[{"field":"event.kind","excluded":false,"queryType":"phrase","value":"signal","valueType":"string"},{"field":"host.name","excluded":false,"queryType":"phrase","value":"{{host.name}}","valueType":"string"}]] +""" +relativeFrom = "now-48h/h" +relativeTo = "now" + +[[transform.investigate]] +label = "Investigate the Subject Process Network Events" +providers = """ +[[{"field":"process.entity_id","excluded":false,"queryType":"phrase","value":"{{process.entity_id}}","valueType":"string"},{"field":"event.category","excluded":false,"queryType":"phrase","value":"network","valueType":"string"}]] +""" + [rule] author = ["Elastic"] @@ -57,12 +79,15 @@ The `MpCmdRun.exe` is a command-line tool part of Windows Defender and is used t - Identify the user account that performed the action and whether it should perform this kind of action. - Contact the account owner and confirm whether they are aware of this activity. - Investigate other alerts associated with the user/host during the past 48 hours. + - $investigate_0 + - $investigate_1 - Check the reputation of the domain or IP address used to host the downloaded file. - Examine the host for derived artifacts that indicate suspicious activities: - Analyze the file using a private sandboxed analysis system. - Observe and collect information about the following activities in both the sandbox and the alert subject host: - Attempts to contact external domains and addresses. - Use the Elastic Defend network events to determine domains and addresses contacted by the subject process by filtering by the process' `process.entity_id`. + - $investigate_2 - Examine the DNS cache for suspicious or anomalous entries. - $osquery_0 - Use the Elastic Defend registry events to examine registry keys accessed, modified, or created by the related processes in the process tree. diff --git a/rules/windows/command_and_control_remote_file_copy_powershell.toml b/rules/windows/command_and_control_remote_file_copy_powershell.toml index 9d38a56ffcf..adf232f29f7 100644 --- a/rules/windows/command_and_control_remote_file_copy_powershell.toml +++ b/rules/windows/command_and_control_remote_file_copy_powershell.toml @@ -2,7 +2,7 @@ creation_date = "2020/11/30" integration = ["endpoint", "windows"] maturity = "production" -updated_date = "2023/06/22" +updated_date = "2023/09/06" min_stack_comments = "New fields added: required_fields, related_integrations, setup" min_stack_version = "8.3.0" @@ -31,6 +31,28 @@ services.path FROM services JOIN authenticode ON services.path = authenticode.pa authenticode.path JOIN hash ON services.path = hash.path WHERE authenticode.result != 'trusted' """ +[[transform.investigate]] +label = "Alerts associated with the user in the last 48h" +providers = """ +[[{"field":"event.kind","excluded":false,"queryType":"phrase","value":"signal","valueType":"string"},{"field":"user.id","excluded":false,"queryType":"phrase","value":"{{user.id}}","valueType":"string"}]] +""" +relativeFrom = "now-48h/h" +relativeTo = "now" + +[[transform.investigate]] +label = "Alerts associated with the host in the last 48h" +providers = """ +[[{"field":"event.kind","excluded":false,"queryType":"phrase","value":"signal","valueType":"string"},{"field":"host.name","excluded":false,"queryType":"phrase","value":"{{host.name}}","valueType":"string"}]] +""" +relativeFrom = "now-48h/h" +relativeTo = "now" + +[[transform.investigate]] +label = "Investigate the Subject Process Network Events" +providers = """ +[[{"field":"process.entity_id","excluded":false,"queryType":"phrase","value":"{{process.entity_id}}","valueType":"string"},{"field":"event.category","excluded":false,"queryType":"phrase","value":"network","valueType":"string"}]] +""" + [rule] author = ["Elastic"] @@ -57,12 +79,15 @@ PowerShell is one of system administrators' main tools for automation, report ro - Identify the user account that performed the action and whether it should perform this kind of action. - Evaluate whether the user needs to use PowerShell to complete tasks. - Investigate other alerts associated with the user/host during the past 48 hours. + - $investigate_0 + - $investigate_1 - Check the reputation of the domain or IP address used to host the downloaded file. - Examine the host for derived artifacts that indicate suspicious activities: - Analyze the file using a private sandboxed analysis system. - Observe and collect information about the following activities in both the sandbox and the alert subject host: - Attempts to contact external domains and addresses. - Use the Elastic Defend network events to determine domains and addresses contacted by the subject process by filtering by the process' `process.entity_id`. + - $investigate_2 - Examine the DNS cache for suspicious or anomalous entries. - $osquery_0 - Use the Elastic Defend registry events to examine registry keys accessed, modified, or created by the related processes in the process tree. From a0d642dcfa2372c1bf4a742f0cdf61b1e747a5ad Mon Sep 17 00:00:00 2001 From: Jonhnathan <26856693+w0rk3r@users.noreply.github.com> Date: Tue, 12 Sep 2023 12:09:56 -0300 Subject: [PATCH 2/9] Add compatibility note --- .../windows/command_and_control_certutil_network_connection.toml | 1 + rules/windows/command_and_control_common_webservices.toml | 1 + .../command_and_control_remote_file_copy_desktopimgdownldr.toml | 1 + rules/windows/command_and_control_remote_file_copy_mpcmdrun.toml | 1 + .../windows/command_and_control_remote_file_copy_powershell.toml | 1 + 5 files changed, 5 insertions(+) diff --git a/rules/windows/command_and_control_certutil_network_connection.toml b/rules/windows/command_and_control_certutil_network_connection.toml index 4ebc7ff6931..377a6726652 100644 --- a/rules/windows/command_and_control_certutil_network_connection.toml +++ b/rules/windows/command_and_control_certutil_network_connection.toml @@ -75,6 +75,7 @@ This rule looks for network events where `certutil.exe` contacts IP ranges other > **Note**: > This investigation guide uses the [Osquery Markdown Plugin](https://www.elastic.co/guide/en/security/master/invest-guide-run-osquery.html) introduced in Elastic Stack version 8.5.0. Older Elastic Stack versions will display unrendered Markdown in this guide. +> This investigation guide uses the [Investigate Markdown Plugin](https://www.elastic.co/guide/en/security/master/interactive-investigation-guides.html) introduced in Elastic Stack version 8.8.0. Older Elastic Stack versions will display unrendered Markdown in this guide. #### Possible investigation steps diff --git a/rules/windows/command_and_control_common_webservices.toml b/rules/windows/command_and_control_common_webservices.toml index b188ec45c41..5e099404525 100644 --- a/rules/windows/command_and_control_common_webservices.toml +++ b/rules/windows/command_and_control_common_webservices.toml @@ -77,6 +77,7 @@ This rule looks for processes outside known legitimate program locations communi > **Note**: > This investigation guide uses the [Osquery Markdown Plugin](https://www.elastic.co/guide/en/security/master/invest-guide-run-osquery.html) introduced in Elastic Stack version 8.5.0. Older Elastic Stack versions will display unrendered Markdown in this guide. +> This investigation guide uses the [Investigate Markdown Plugin](https://www.elastic.co/guide/en/security/master/interactive-investigation-guides.html) introduced in Elastic Stack version 8.8.0. Older Elastic Stack versions will display unrendered Markdown in this guide. #### Possible investigation steps diff --git a/rules/windows/command_and_control_remote_file_copy_desktopimgdownldr.toml b/rules/windows/command_and_control_remote_file_copy_desktopimgdownldr.toml index 084da1f0178..743a0638ecc 100644 --- a/rules/windows/command_and_control_remote_file_copy_desktopimgdownldr.toml +++ b/rules/windows/command_and_control_remote_file_copy_desktopimgdownldr.toml @@ -75,6 +75,7 @@ The `Desktopimgdownldr.exe` utility is used to to configure lockscreen/desktop i > **Note**: > This investigation guide uses the [Osquery Markdown Plugin](https://www.elastic.co/guide/en/security/master/invest-guide-run-osquery.html) introduced in Elastic Stack version 8.5.0. Older Elastic Stack versions will display unrendered Markdown in this guide. +> This investigation guide uses the [Investigate Markdown Plugin](https://www.elastic.co/guide/en/security/master/interactive-investigation-guides.html) introduced in Elastic Stack version 8.8.0. Older Elastic Stack versions will display unrendered Markdown in this guide. #### Possible investigation steps diff --git a/rules/windows/command_and_control_remote_file_copy_mpcmdrun.toml b/rules/windows/command_and_control_remote_file_copy_mpcmdrun.toml index 28255cc86f7..fed4335e32e 100644 --- a/rules/windows/command_and_control_remote_file_copy_mpcmdrun.toml +++ b/rules/windows/command_and_control_remote_file_copy_mpcmdrun.toml @@ -72,6 +72,7 @@ The `MpCmdRun.exe` is a command-line tool part of Windows Defender and is used t > **Note**: > This investigation guide uses the [Osquery Markdown Plugin](https://www.elastic.co/guide/en/security/master/invest-guide-run-osquery.html) introduced in Elastic Stack version 8.5.0. Older Elastic Stack versions will display unrendered Markdown in this guide. +> This investigation guide uses the [Investigate Markdown Plugin](https://www.elastic.co/guide/en/security/master/interactive-investigation-guides.html) introduced in Elastic Stack version 8.8.0. Older Elastic Stack versions will display unrendered Markdown in this guide. #### Possible investigation steps diff --git a/rules/windows/command_and_control_remote_file_copy_powershell.toml b/rules/windows/command_and_control_remote_file_copy_powershell.toml index adf232f29f7..b6c1053e678 100644 --- a/rules/windows/command_and_control_remote_file_copy_powershell.toml +++ b/rules/windows/command_and_control_remote_file_copy_powershell.toml @@ -72,6 +72,7 @@ PowerShell is one of system administrators' main tools for automation, report ro > **Note**: > This investigation guide uses the [Osquery Markdown Plugin](https://www.elastic.co/guide/en/security/master/invest-guide-run-osquery.html) introduced in Elastic Stack version 8.5.0. Older Elastic Stack versions will display unrendered Markdown in this guide. +> This investigation guide uses the [Investigate Markdown Plugin](https://www.elastic.co/guide/en/security/master/interactive-investigation-guides.html) introduced in Elastic Stack version 8.8.0. Older Elastic Stack versions will display unrendered Markdown in this guide. #### Possible investigation steps From 0f09f053ff4bcda1b83197b60f99f8e418bb672d Mon Sep 17 00:00:00 2001 From: Jonhnathan <26856693+w0rk3r@users.noreply.github.com> Date: Wed, 13 Sep 2023 13:48:23 -0300 Subject: [PATCH 3/9] Update Transform format --- ...d_control_certutil_network_connection.toml | 56 +++++++++++++++--- ...ommand_and_control_common_webservices.toml | 57 ++++++++++++++++--- ...ol_remote_file_copy_desktopimgdownldr.toml | 57 ++++++++++++++++--- ...and_control_remote_file_copy_mpcmdrun.toml | 57 ++++++++++++++++--- ...d_control_remote_file_copy_powershell.toml | 57 ++++++++++++++++--- 5 files changed, 239 insertions(+), 45 deletions(-) diff --git a/rules/windows/command_and_control_certutil_network_connection.toml b/rules/windows/command_and_control_certutil_network_connection.toml index 377a6726652..0607de69800 100644 --- a/rules/windows/command_and_control_certutil_network_connection.toml +++ b/rules/windows/command_and_control_certutil_network_connection.toml @@ -33,26 +33,64 @@ authenticode.path JOIN hash ON services.path = hash.path WHERE authenticode.resu [[transform.investigate]] label = "Alerts associated with the user in the last 48h" -providers = """ -[[{"field":"event.kind","excluded":false,"queryType":"phrase","value":"signal","valueType":"string"},{"field":"user.id","excluded":false,"queryType":"phrase","value":"{{user.id}}","valueType":"string"}]] -""" relativeFrom = "now-48h/h" relativeTo = "now" +[transform.investigate.providers] + +[[transform.investigate.providers.field_info]] +field = "event.kind" +excluded = false +queryType = "phrase" +value = "signal" +valueType = "string" + +[[transform.investigate.providers.field_info]] +field = "user.id" +excluded = false +queryType = "phrase" +value = "{{user.id}}" +valueType = "string" + [[transform.investigate]] label = "Alerts associated with the host in the last 48h" -providers = """ -[[{"field":"event.kind","excluded":false,"queryType":"phrase","value":"signal","valueType":"string"},{"field":"host.name","excluded":false,"queryType":"phrase","value":"{{host.name}}","valueType":"string"}]] -""" relativeFrom = "now-48h/h" relativeTo = "now" +[transform.investigate.providers] + +[[transform.investigate.providers.field_info]] +field = "event.kind" +excluded = false +queryType = "phrase" +value = "signal" +valueType = "string" + +[[transform.investigate.providers.field_info]] +field = "host.name" +excluded = false +queryType = "phrase" +value = "{{host.name}}" +valueType = "string" + [[transform.investigate]] label = "Investigate the Subject Process Network Events" -providers = """ -[[{"field":"process.entity_id","excluded":false,"queryType":"phrase","value":"{{process.entity_id}}","valueType":"string"},{"field":"event.category","excluded":false,"queryType":"phrase","value":"network","valueType":"string"}]] -""" +[transform.investigate.providers] + +[[transform.investigate.providers.field_info]] +field = "process.entity_id" +excluded = false +queryType = "phrase" +value = "{{process.entity_id}}" +valueType = "string" + +[[transform.investigate.providers.field_info]] +field = "event.category" +excluded = false +queryType = "phrase" +value = "network" +valueType = "string" [rule] author = ["Elastic"] diff --git a/rules/windows/command_and_control_common_webservices.toml b/rules/windows/command_and_control_common_webservices.toml index 5e099404525..ece84458ce9 100644 --- a/rules/windows/command_and_control_common_webservices.toml +++ b/rules/windows/command_and_control_common_webservices.toml @@ -33,25 +33,64 @@ authenticode.path JOIN hash ON services.path = hash.path WHERE authenticode.resu [[transform.investigate]] label = "Alerts associated with the user in the last 48h" -providers = """ -[[{"field":"event.kind","excluded":false,"queryType":"phrase","value":"signal","valueType":"string"},{"field":"user.id","excluded":false,"queryType":"phrase","value":"{{user.id}}","valueType":"string"}]] -""" relativeFrom = "now-48h/h" relativeTo = "now" +[transform.investigate.providers] + +[[transform.investigate.providers.field_info]] +field = "event.kind" +excluded = false +queryType = "phrase" +value = "signal" +valueType = "string" + +[[transform.investigate.providers.field_info]] +field = "user.id" +excluded = false +queryType = "phrase" +value = "{{user.id}}" +valueType = "string" + [[transform.investigate]] label = "Alerts associated with the host in the last 48h" -providers = """ -[[{"field":"event.kind","excluded":false,"queryType":"phrase","value":"signal","valueType":"string"},{"field":"host.name","excluded":false,"queryType":"phrase","value":"{{host.name}}","valueType":"string"}]] -""" relativeFrom = "now-48h/h" relativeTo = "now" +[transform.investigate.providers] + +[[transform.investigate.providers.field_info]] +field = "event.kind" +excluded = false +queryType = "phrase" +value = "signal" +valueType = "string" + +[[transform.investigate.providers.field_info]] +field = "host.name" +excluded = false +queryType = "phrase" +value = "{{host.name}}" +valueType = "string" + [[transform.investigate]] label = "Investigate the Subject Process Network Events" -providers = """ -[[{"field":"process.entity_id","excluded":false,"queryType":"phrase","value":"{{process.entity_id}}","valueType":"string"},{"field":"event.category","excluded":false,"queryType":"phrase","value":"network","valueType":"string"}]] -""" + +[transform.investigate.providers] + +[[transform.investigate.providers.field_info]] +field = "process.entity_id" +excluded = false +queryType = "phrase" +value = "{{process.entity_id}}" +valueType = "string" + +[[transform.investigate.providers.field_info]] +field = "event.category" +excluded = false +queryType = "phrase" +value = "network" +valueType = "string" [rule] diff --git a/rules/windows/command_and_control_remote_file_copy_desktopimgdownldr.toml b/rules/windows/command_and_control_remote_file_copy_desktopimgdownldr.toml index 743a0638ecc..6e6cd1f64cb 100644 --- a/rules/windows/command_and_control_remote_file_copy_desktopimgdownldr.toml +++ b/rules/windows/command_and_control_remote_file_copy_desktopimgdownldr.toml @@ -33,25 +33,64 @@ authenticode.path JOIN hash ON services.path = hash.path WHERE authenticode.resu [[transform.investigate]] label = "Alerts associated with the user in the last 48h" -providers = """ -[[{"field":"event.kind","excluded":false,"queryType":"phrase","value":"signal","valueType":"string"},{"field":"user.id","excluded":false,"queryType":"phrase","value":"{{user.id}}","valueType":"string"}]] -""" relativeFrom = "now-48h/h" relativeTo = "now" +[transform.investigate.providers] + +[[transform.investigate.providers.field_info]] +field = "event.kind" +excluded = false +queryType = "phrase" +value = "signal" +valueType = "string" + +[[transform.investigate.providers.field_info]] +field = "user.id" +excluded = false +queryType = "phrase" +value = "{{user.id}}" +valueType = "string" + [[transform.investigate]] label = "Alerts associated with the host in the last 48h" -providers = """ -[[{"field":"event.kind","excluded":false,"queryType":"phrase","value":"signal","valueType":"string"},{"field":"host.name","excluded":false,"queryType":"phrase","value":"{{host.name}}","valueType":"string"}]] -""" relativeFrom = "now-48h/h" relativeTo = "now" +[transform.investigate.providers] + +[[transform.investigate.providers.field_info]] +field = "event.kind" +excluded = false +queryType = "phrase" +value = "signal" +valueType = "string" + +[[transform.investigate.providers.field_info]] +field = "host.name" +excluded = false +queryType = "phrase" +value = "{{host.name}}" +valueType = "string" + [[transform.investigate]] label = "Investigate the Subject Process Network Events" -providers = """ -[[{"field":"process.entity_id","excluded":false,"queryType":"phrase","value":"{{process.entity_id}}","valueType":"string"},{"field":"event.category","excluded":false,"queryType":"phrase","value":"network","valueType":"string"}]] -""" + +[transform.investigate.providers] + +[[transform.investigate.providers.field_info]] +field = "process.entity_id" +excluded = false +queryType = "phrase" +value = "{{process.entity_id}}" +valueType = "string" + +[[transform.investigate.providers.field_info]] +field = "event.category" +excluded = false +queryType = "phrase" +value = "network" +valueType = "string" [rule] diff --git a/rules/windows/command_and_control_remote_file_copy_mpcmdrun.toml b/rules/windows/command_and_control_remote_file_copy_mpcmdrun.toml index fed4335e32e..56bb2cef33f 100644 --- a/rules/windows/command_and_control_remote_file_copy_mpcmdrun.toml +++ b/rules/windows/command_and_control_remote_file_copy_mpcmdrun.toml @@ -33,25 +33,64 @@ authenticode.path JOIN hash ON services.path = hash.path WHERE authenticode.resu [[transform.investigate]] label = "Alerts associated with the user in the last 48h" -providers = """ -[[{"field":"event.kind","excluded":false,"queryType":"phrase","value":"signal","valueType":"string"},{"field":"user.id","excluded":false,"queryType":"phrase","value":"{{user.id}}","valueType":"string"}]] -""" relativeFrom = "now-48h/h" relativeTo = "now" +[transform.investigate.providers] + +[[transform.investigate.providers.field_info]] +field = "event.kind" +excluded = false +queryType = "phrase" +value = "signal" +valueType = "string" + +[[transform.investigate.providers.field_info]] +field = "user.id" +excluded = false +queryType = "phrase" +value = "{{user.id}}" +valueType = "string" + [[transform.investigate]] label = "Alerts associated with the host in the last 48h" -providers = """ -[[{"field":"event.kind","excluded":false,"queryType":"phrase","value":"signal","valueType":"string"},{"field":"host.name","excluded":false,"queryType":"phrase","value":"{{host.name}}","valueType":"string"}]] -""" relativeFrom = "now-48h/h" relativeTo = "now" +[transform.investigate.providers] + +[[transform.investigate.providers.field_info]] +field = "event.kind" +excluded = false +queryType = "phrase" +value = "signal" +valueType = "string" + +[[transform.investigate.providers.field_info]] +field = "host.name" +excluded = false +queryType = "phrase" +value = "{{host.name}}" +valueType = "string" + [[transform.investigate]] label = "Investigate the Subject Process Network Events" -providers = """ -[[{"field":"process.entity_id","excluded":false,"queryType":"phrase","value":"{{process.entity_id}}","valueType":"string"},{"field":"event.category","excluded":false,"queryType":"phrase","value":"network","valueType":"string"}]] -""" + +[transform.investigate.providers] + +[[transform.investigate.providers.field_info]] +field = "process.entity_id" +excluded = false +queryType = "phrase" +value = "{{process.entity_id}}" +valueType = "string" + +[[transform.investigate.providers.field_info]] +field = "event.category" +excluded = false +queryType = "phrase" +value = "network" +valueType = "string" [rule] diff --git a/rules/windows/command_and_control_remote_file_copy_powershell.toml b/rules/windows/command_and_control_remote_file_copy_powershell.toml index b6c1053e678..b2e8403ab40 100644 --- a/rules/windows/command_and_control_remote_file_copy_powershell.toml +++ b/rules/windows/command_and_control_remote_file_copy_powershell.toml @@ -33,25 +33,64 @@ authenticode.path JOIN hash ON services.path = hash.path WHERE authenticode.resu [[transform.investigate]] label = "Alerts associated with the user in the last 48h" -providers = """ -[[{"field":"event.kind","excluded":false,"queryType":"phrase","value":"signal","valueType":"string"},{"field":"user.id","excluded":false,"queryType":"phrase","value":"{{user.id}}","valueType":"string"}]] -""" relativeFrom = "now-48h/h" relativeTo = "now" +[transform.investigate.providers] + +[[transform.investigate.providers.field_info]] +field = "event.kind" +excluded = false +queryType = "phrase" +value = "signal" +valueType = "string" + +[[transform.investigate.providers.field_info]] +field = "user.id" +excluded = false +queryType = "phrase" +value = "{{user.id}}" +valueType = "string" + [[transform.investigate]] label = "Alerts associated with the host in the last 48h" -providers = """ -[[{"field":"event.kind","excluded":false,"queryType":"phrase","value":"signal","valueType":"string"},{"field":"host.name","excluded":false,"queryType":"phrase","value":"{{host.name}}","valueType":"string"}]] -""" relativeFrom = "now-48h/h" relativeTo = "now" +[transform.investigate.providers] + +[[transform.investigate.providers.field_info]] +field = "event.kind" +excluded = false +queryType = "phrase" +value = "signal" +valueType = "string" + +[[transform.investigate.providers.field_info]] +field = "host.name" +excluded = false +queryType = "phrase" +value = "{{host.name}}" +valueType = "string" + [[transform.investigate]] label = "Investigate the Subject Process Network Events" -providers = """ -[[{"field":"process.entity_id","excluded":false,"queryType":"phrase","value":"{{process.entity_id}}","valueType":"string"},{"field":"event.category","excluded":false,"queryType":"phrase","value":"network","valueType":"string"}]] -""" + +[transform.investigate.providers] + +[[transform.investigate.providers.field_info]] +field = "process.entity_id" +excluded = false +queryType = "phrase" +value = "{{process.entity_id}}" +valueType = "string" + +[[transform.investigate.providers.field_info]] +field = "event.category" +excluded = false +queryType = "phrase" +value = "network" +valueType = "string" [rule] From 5c28b5afb401c1dd7dde0b62ad466ec9a0006995 Mon Sep 17 00:00:00 2001 From: brokensound77 Date: Wed, 13 Sep 2023 20:22:08 -0600 Subject: [PATCH 4/9] add investigate to Transform schema --- detection_rules/rule.py | 28 +++++++++++--------------- detection_rules/schemas/definitions.py | 4 ++++ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/detection_rules/rule.py b/detection_rules/rule.py index 177f51dd11e..503c0ae35e7 100644 --- a/detection_rules/rule.py +++ b/detection_rules/rule.py @@ -91,34 +91,30 @@ class OsQuery: ecs_mapping: Optional[Dict[str, Dict[Literal['field', 'value'], str]]] @dataclass(frozen=True) - class Insight: + class Investigate: @dataclass(frozen=True) class Provider: + excluded: definitions.InvestigateProviderExcluded field: str + queryType: definitions.InvestigateProviderQueryType value: str - type: str + ValueType: definitions.InvestigateProviderValueType label: str + description: str providers: List[List[Provider]] + relativeFrom: Optional[str] + relativeTo: Optional[str] # these must be lists in order to have more than one. Their index in the list is how they will be referenced in the # note string templates osquery: Optional[List[OsQuery]] - insight: Optional[List[Insight]] + investigate: Optional[List[Investigate]] - @validates_schema - def validate_transforms(self, value: dict, **kwargs): - """Validate transform fields.""" - # temporarily invalidate insights until schema stabilizes - insight = value.get('insight') - if insight is not None: - raise NotImplementedError('Insights are not stable yet.') - return - - def render_insight_osquery_to_string(self) -> Dict[Literal['osquery', 'insight'], List[str]]: + def render_investigate_osquery_to_string(self) -> Dict[definitions.TransformTypes, List[str]]: obj = self.to_dict() - rendered: Dict[Literal['osquery', 'insight'], List[str]] = {'osquery': [], 'insight': []} + rendered: Dict[definitions.TransformTypes, List[str]] = {'osquery': [], 'investigate': []} for plugin, entries in obj.items(): for entry in entries: rendered[plugin].append(f'!{{{plugin}{json.dumps(entry, sort_keys=True, separators=(",", ":"))}}}') @@ -343,12 +339,12 @@ def process_transforms(cls, transform: RuleTransform, obj: dict) -> dict: # only create functions that CAREFULLY mutate the obj dict def process_note_plugins(): - """Format the note field with osquery and insight plugin strings.""" + """Format the note field with osquery and investigate plugin strings.""" note = obj.get('note') if not note: return - rendered = transform.render_insight_osquery_to_string() + rendered = transform.render_investigate_osquery_to_string() rendered_patterns = {} for plugin, entries in rendered.items(): rendered_patterns.update(**{f'{plugin}_{i}': e for i, e in enumerate(entries)}) diff --git a/detection_rules/schemas/definitions.py b/detection_rules/schemas/definitions.py index f6d56a322aa..698520709fe 100644 --- a/detection_rules/schemas/definitions.py +++ b/detection_rules/schemas/definitions.py @@ -131,6 +131,9 @@ Date = NewType('Date', str, validate=validate.Regexp(DATE_PATTERN)) FilterLanguages = Literal["kuery", "lucene"] Interval = NewType('Interval', str, validate=validate.Regexp(INTERVAL_PATTERN)) +InvestigateProviderExcluded = Literal["is", "is not", "is one of", "is not one of", "exists", "does not exist"] +InvestigateProviderQueryType = Literal["phrase", "range"] +InvestigateProviderValueType = Literal["string", "boolean"] Markdown = NewType("MarkdownField", CodeString) Maturity = Literal['development', 'experimental', 'beta', 'production', 'deprecated'] MaxSignals = NewType("MaxSignals", int, validate=validate.Range(min=1)) @@ -151,6 +154,7 @@ ThresholdValue = NewType("ThresholdValue", int, validate=validate.Range(min=1)) TimelineTemplateId = NewType('TimelineTemplateId', str, validate=validate.OneOf(list(TIMELINE_TEMPLATES))) TimelineTemplateTitle = NewType('TimelineTemplateTitle', str, validate=validate.OneOf(TIMELINE_TEMPLATES.values())) +TransformTypes = Literal["osquery", "investigate"] UUIDString = NewType('UUIDString', str, validate=validate.Regexp(UUID_PATTERN)) BuildingBlockType = Literal['default'] From 409c4d9a882245e6c1212cd9cf3f8b728c3d2f2e Mon Sep 17 00:00:00 2001 From: Jonhnathan <26856693+w0rk3r@users.noreply.github.com> Date: Thu, 7 Dec 2023 17:44:10 -0300 Subject: [PATCH 5/9] Drop .field_info --- .../command_and_control_common_webservices.toml | 12 ++++++------ ...d_control_remote_file_copy_desktopimgdownldr.toml | 12 ++++++------ ...ommand_and_control_remote_file_copy_mpcmdrun.toml | 12 ++++++------ ...mand_and_control_remote_file_copy_powershell.toml | 12 ++++++------ ...mand_and_control_certutil_network_connection.toml | 12 ++++++------ 5 files changed, 30 insertions(+), 30 deletions(-) diff --git a/rules/windows/command_and_control_common_webservices.toml b/rules/windows/command_and_control_common_webservices.toml index a95ff89879f..2b8dcb02e9c 100644 --- a/rules/windows/command_and_control_common_webservices.toml +++ b/rules/windows/command_and_control_common_webservices.toml @@ -38,14 +38,14 @@ relativeTo = "now" [transform.investigate.providers] -[[transform.investigate.providers.field_info]] +[[transform.investigate.providers]] field = "event.kind" excluded = false queryType = "phrase" value = "signal" valueType = "string" -[[transform.investigate.providers.field_info]] +[[transform.investigate.providers]] field = "user.id" excluded = false queryType = "phrase" @@ -59,14 +59,14 @@ relativeTo = "now" [transform.investigate.providers] -[[transform.investigate.providers.field_info]] +[[transform.investigate.providers]] field = "event.kind" excluded = false queryType = "phrase" value = "signal" valueType = "string" -[[transform.investigate.providers.field_info]] +[[transform.investigate.providers]] field = "host.name" excluded = false queryType = "phrase" @@ -78,14 +78,14 @@ label = "Investigate the Subject Process Network Events" [transform.investigate.providers] -[[transform.investigate.providers.field_info]] +[[transform.investigate.providers]] field = "process.entity_id" excluded = false queryType = "phrase" value = "{{process.entity_id}}" valueType = "string" -[[transform.investigate.providers.field_info]] +[[transform.investigate.providers]] field = "event.category" excluded = false queryType = "phrase" diff --git a/rules/windows/command_and_control_remote_file_copy_desktopimgdownldr.toml b/rules/windows/command_and_control_remote_file_copy_desktopimgdownldr.toml index 108ffbbf287..d854694772e 100644 --- a/rules/windows/command_and_control_remote_file_copy_desktopimgdownldr.toml +++ b/rules/windows/command_and_control_remote_file_copy_desktopimgdownldr.toml @@ -38,14 +38,14 @@ relativeTo = "now" [transform.investigate.providers] -[[transform.investigate.providers.field_info]] +[[transform.investigate.providers]] field = "event.kind" excluded = false queryType = "phrase" value = "signal" valueType = "string" -[[transform.investigate.providers.field_info]] +[[transform.investigate.providers]] field = "user.id" excluded = false queryType = "phrase" @@ -59,14 +59,14 @@ relativeTo = "now" [transform.investigate.providers] -[[transform.investigate.providers.field_info]] +[[transform.investigate.providers]] field = "event.kind" excluded = false queryType = "phrase" value = "signal" valueType = "string" -[[transform.investigate.providers.field_info]] +[[transform.investigate.providers]] field = "host.name" excluded = false queryType = "phrase" @@ -78,14 +78,14 @@ label = "Investigate the Subject Process Network Events" [transform.investigate.providers] -[[transform.investigate.providers.field_info]] +[[transform.investigate.providers]] field = "process.entity_id" excluded = false queryType = "phrase" value = "{{process.entity_id}}" valueType = "string" -[[transform.investigate.providers.field_info]] +[[transform.investigate.providers]] field = "event.category" excluded = false queryType = "phrase" diff --git a/rules/windows/command_and_control_remote_file_copy_mpcmdrun.toml b/rules/windows/command_and_control_remote_file_copy_mpcmdrun.toml index a200453b4f1..7db4435945e 100644 --- a/rules/windows/command_and_control_remote_file_copy_mpcmdrun.toml +++ b/rules/windows/command_and_control_remote_file_copy_mpcmdrun.toml @@ -38,14 +38,14 @@ relativeTo = "now" [transform.investigate.providers] -[[transform.investigate.providers.field_info]] +[[transform.investigate.providers]] field = "event.kind" excluded = false queryType = "phrase" value = "signal" valueType = "string" -[[transform.investigate.providers.field_info]] +[[transform.investigate.providers]] field = "user.id" excluded = false queryType = "phrase" @@ -59,14 +59,14 @@ relativeTo = "now" [transform.investigate.providers] -[[transform.investigate.providers.field_info]] +[[transform.investigate.providers]] field = "event.kind" excluded = false queryType = "phrase" value = "signal" valueType = "string" -[[transform.investigate.providers.field_info]] +[[transform.investigate.providers]] field = "host.name" excluded = false queryType = "phrase" @@ -78,14 +78,14 @@ label = "Investigate the Subject Process Network Events" [transform.investigate.providers] -[[transform.investigate.providers.field_info]] +[[transform.investigate.providers]] field = "process.entity_id" excluded = false queryType = "phrase" value = "{{process.entity_id}}" valueType = "string" -[[transform.investigate.providers.field_info]] +[[transform.investigate.providers]] field = "event.category" excluded = false queryType = "phrase" diff --git a/rules/windows/command_and_control_remote_file_copy_powershell.toml b/rules/windows/command_and_control_remote_file_copy_powershell.toml index a27057811eb..9595dcc36f5 100644 --- a/rules/windows/command_and_control_remote_file_copy_powershell.toml +++ b/rules/windows/command_and_control_remote_file_copy_powershell.toml @@ -38,14 +38,14 @@ relativeTo = "now" [transform.investigate.providers] -[[transform.investigate.providers.field_info]] +[[transform.investigate.providers]] field = "event.kind" excluded = false queryType = "phrase" value = "signal" valueType = "string" -[[transform.investigate.providers.field_info]] +[[transform.investigate.providers]] field = "user.id" excluded = false queryType = "phrase" @@ -59,14 +59,14 @@ relativeTo = "now" [transform.investigate.providers] -[[transform.investigate.providers.field_info]] +[[transform.investigate.providers]] field = "event.kind" excluded = false queryType = "phrase" value = "signal" valueType = "string" -[[transform.investigate.providers.field_info]] +[[transform.investigate.providers]] field = "host.name" excluded = false queryType = "phrase" @@ -78,14 +78,14 @@ label = "Investigate the Subject Process Network Events" [transform.investigate.providers] -[[transform.investigate.providers.field_info]] +[[transform.investigate.providers]] field = "process.entity_id" excluded = false queryType = "phrase" value = "{{process.entity_id}}" valueType = "string" -[[transform.investigate.providers.field_info]] +[[transform.investigate.providers]] field = "event.category" excluded = false queryType = "phrase" diff --git a/rules_building_block/command_and_control_certutil_network_connection.toml b/rules_building_block/command_and_control_certutil_network_connection.toml index 96bf270238d..5442e8255c6 100644 --- a/rules_building_block/command_and_control_certutil_network_connection.toml +++ b/rules_building_block/command_and_control_certutil_network_connection.toml @@ -39,14 +39,14 @@ relativeTo = "now" [transform.investigate.providers] -[[transform.investigate.providers.field_info]] +[[transform.investigate.providers]] field = "event.kind" excluded = false queryType = "phrase" value = "signal" valueType = "string" -[[transform.investigate.providers.field_info]] +[[transform.investigate.providers]] field = "user.id" excluded = false queryType = "phrase" @@ -60,14 +60,14 @@ relativeTo = "now" [transform.investigate.providers] -[[transform.investigate.providers.field_info]] +[[transform.investigate.providers]] field = "event.kind" excluded = false queryType = "phrase" value = "signal" valueType = "string" -[[transform.investigate.providers.field_info]] +[[transform.investigate.providers]] field = "host.name" excluded = false queryType = "phrase" @@ -79,14 +79,14 @@ label = "Investigate the Subject Process Network Events" [transform.investigate.providers] -[[transform.investigate.providers.field_info]] +[[transform.investigate.providers]] field = "process.entity_id" excluded = false queryType = "phrase" value = "{{process.entity_id}}" valueType = "string" -[[transform.investigate.providers.field_info]] +[[transform.investigate.providers]] field = "event.category" excluded = false queryType = "phrase" From 43971f6cf08db6813b54a3d384221af807afbbdb Mon Sep 17 00:00:00 2001 From: brokensound77 Date: Thu, 7 Dec 2023 13:54:38 -0700 Subject: [PATCH 6/9] fix investigate toml --- rules/windows/command_and_control_common_webservices.toml | 6 ------ ...mand_and_control_remote_file_copy_desktopimgdownldr.toml | 6 ------ .../command_and_control_remote_file_copy_mpcmdrun.toml | 6 ------ .../command_and_control_remote_file_copy_powershell.toml | 6 ------ .../command_and_control_certutil_network_connection.toml | 5 ----- 5 files changed, 29 deletions(-) diff --git a/rules/windows/command_and_control_common_webservices.toml b/rules/windows/command_and_control_common_webservices.toml index 2b8dcb02e9c..81038a98cba 100644 --- a/rules/windows/command_and_control_common_webservices.toml +++ b/rules/windows/command_and_control_common_webservices.toml @@ -36,8 +36,6 @@ label = "Alerts associated with the user in the last 48h" relativeFrom = "now-48h/h" relativeTo = "now" -[transform.investigate.providers] - [[transform.investigate.providers]] field = "event.kind" excluded = false @@ -57,8 +55,6 @@ label = "Alerts associated with the host in the last 48h" relativeFrom = "now-48h/h" relativeTo = "now" -[transform.investigate.providers] - [[transform.investigate.providers]] field = "event.kind" excluded = false @@ -76,8 +72,6 @@ valueType = "string" [[transform.investigate]] label = "Investigate the Subject Process Network Events" -[transform.investigate.providers] - [[transform.investigate.providers]] field = "process.entity_id" excluded = false diff --git a/rules/windows/command_and_control_remote_file_copy_desktopimgdownldr.toml b/rules/windows/command_and_control_remote_file_copy_desktopimgdownldr.toml index d854694772e..dafbd878f30 100644 --- a/rules/windows/command_and_control_remote_file_copy_desktopimgdownldr.toml +++ b/rules/windows/command_and_control_remote_file_copy_desktopimgdownldr.toml @@ -36,8 +36,6 @@ label = "Alerts associated with the user in the last 48h" relativeFrom = "now-48h/h" relativeTo = "now" -[transform.investigate.providers] - [[transform.investigate.providers]] field = "event.kind" excluded = false @@ -57,8 +55,6 @@ label = "Alerts associated with the host in the last 48h" relativeFrom = "now-48h/h" relativeTo = "now" -[transform.investigate.providers] - [[transform.investigate.providers]] field = "event.kind" excluded = false @@ -76,8 +72,6 @@ valueType = "string" [[transform.investigate]] label = "Investigate the Subject Process Network Events" -[transform.investigate.providers] - [[transform.investigate.providers]] field = "process.entity_id" excluded = false diff --git a/rules/windows/command_and_control_remote_file_copy_mpcmdrun.toml b/rules/windows/command_and_control_remote_file_copy_mpcmdrun.toml index 7db4435945e..11d24a86486 100644 --- a/rules/windows/command_and_control_remote_file_copy_mpcmdrun.toml +++ b/rules/windows/command_and_control_remote_file_copy_mpcmdrun.toml @@ -36,8 +36,6 @@ label = "Alerts associated with the user in the last 48h" relativeFrom = "now-48h/h" relativeTo = "now" -[transform.investigate.providers] - [[transform.investigate.providers]] field = "event.kind" excluded = false @@ -57,8 +55,6 @@ label = "Alerts associated with the host in the last 48h" relativeFrom = "now-48h/h" relativeTo = "now" -[transform.investigate.providers] - [[transform.investigate.providers]] field = "event.kind" excluded = false @@ -76,8 +72,6 @@ valueType = "string" [[transform.investigate]] label = "Investigate the Subject Process Network Events" -[transform.investigate.providers] - [[transform.investigate.providers]] field = "process.entity_id" excluded = false diff --git a/rules/windows/command_and_control_remote_file_copy_powershell.toml b/rules/windows/command_and_control_remote_file_copy_powershell.toml index 9595dcc36f5..740ffa2b026 100644 --- a/rules/windows/command_and_control_remote_file_copy_powershell.toml +++ b/rules/windows/command_and_control_remote_file_copy_powershell.toml @@ -36,8 +36,6 @@ label = "Alerts associated with the user in the last 48h" relativeFrom = "now-48h/h" relativeTo = "now" -[transform.investigate.providers] - [[transform.investigate.providers]] field = "event.kind" excluded = false @@ -57,8 +55,6 @@ label = "Alerts associated with the host in the last 48h" relativeFrom = "now-48h/h" relativeTo = "now" -[transform.investigate.providers] - [[transform.investigate.providers]] field = "event.kind" excluded = false @@ -76,8 +72,6 @@ valueType = "string" [[transform.investigate]] label = "Investigate the Subject Process Network Events" -[transform.investigate.providers] - [[transform.investigate.providers]] field = "process.entity_id" excluded = false diff --git a/rules_building_block/command_and_control_certutil_network_connection.toml b/rules_building_block/command_and_control_certutil_network_connection.toml index 5442e8255c6..d55645b0e8c 100644 --- a/rules_building_block/command_and_control_certutil_network_connection.toml +++ b/rules_building_block/command_and_control_certutil_network_connection.toml @@ -37,7 +37,6 @@ label = "Alerts associated with the user in the last 48h" relativeFrom = "now-48h/h" relativeTo = "now" -[transform.investigate.providers] [[transform.investigate.providers]] field = "event.kind" @@ -58,8 +57,6 @@ label = "Alerts associated with the host in the last 48h" relativeFrom = "now-48h/h" relativeTo = "now" -[transform.investigate.providers] - [[transform.investigate.providers]] field = "event.kind" excluded = false @@ -77,8 +74,6 @@ valueType = "string" [[transform.investigate]] label = "Investigate the Subject Process Network Events" -[transform.investigate.providers] - [[transform.investigate.providers]] field = "process.entity_id" excluded = false From bfc6e61fe4d5ba751584a942e563954ca7260b26 Mon Sep 17 00:00:00 2001 From: brokensound77 Date: Thu, 7 Dec 2023 19:46:06 -0700 Subject: [PATCH 7/9] repair rule formats and update schema --- detection_rules/rule.py | 6 +- detection_rules/schemas/definitions.py | 1 - ...ommand_and_control_common_webservices.toml | 59 ++++++------------ ...ol_remote_file_copy_desktopimgdownldr.toml | 60 ++++++------------- ...and_control_remote_file_copy_mpcmdrun.toml | 60 ++++++------------- ...d_control_remote_file_copy_powershell.toml | 60 ++++++------------- ...d_control_certutil_network_connection.toml | 60 ++++++------------- 7 files changed, 93 insertions(+), 213 deletions(-) diff --git a/detection_rules/rule.py b/detection_rules/rule.py index b994815f3e1..d63619a2e4d 100644 --- a/detection_rules/rule.py +++ b/detection_rules/rule.py @@ -94,14 +94,14 @@ class OsQuery: class Investigate: @dataclass(frozen=True) class Provider: - excluded: definitions.InvestigateProviderExcluded + excluded: bool field: str queryType: definitions.InvestigateProviderQueryType value: str - ValueType: definitions.InvestigateProviderValueType + valueType: definitions.InvestigateProviderValueType label: str - description: str + description: Optional[str] providers: List[List[Provider]] relativeFrom: Optional[str] relativeTo: Optional[str] diff --git a/detection_rules/schemas/definitions.py b/detection_rules/schemas/definitions.py index 9a3f748cfd4..e1b1c5273a3 100644 --- a/detection_rules/schemas/definitions.py +++ b/detection_rules/schemas/definitions.py @@ -140,7 +140,6 @@ Date = NewType('Date', str, validate=validate.Regexp(DATE_PATTERN)) FilterLanguages = Literal["kuery", "lucene", "eql", "esql"] Interval = NewType('Interval', str, validate=validate.Regexp(INTERVAL_PATTERN)) -InvestigateProviderExcluded = Literal["is", "is not", "is one of", "is not one of", "exists", "does not exist"] InvestigateProviderQueryType = Literal["phrase", "range"] InvestigateProviderValueType = Literal["string", "boolean"] Markdown = NewType("MarkdownField", CodeString) diff --git a/rules/windows/command_and_control_common_webservices.toml b/rules/windows/command_and_control_common_webservices.toml index 81038a98cba..4c3bd0d90a0 100644 --- a/rules/windows/command_and_control_common_webservices.toml +++ b/rules/windows/command_and_control_common_webservices.toml @@ -35,56 +35,33 @@ authenticode.path JOIN hash ON services.path = hash.path WHERE authenticode.resu label = "Alerts associated with the user in the last 48h" relativeFrom = "now-48h/h" relativeTo = "now" +providers = [ + [ + {field = "event.kind", excluded = false, queryType = "phrase", value = "signal", valueType = "string"}, + {field = "user.id", excluded = false, queryType = "phrase", value = "{{user.id}}", valueType = "string"} + ] +] -[[transform.investigate.providers]] -field = "event.kind" -excluded = false -queryType = "phrase" -value = "signal" -valueType = "string" - -[[transform.investigate.providers]] -field = "user.id" -excluded = false -queryType = "phrase" -value = "{{user.id}}" -valueType = "string" [[transform.investigate]] label = "Alerts associated with the host in the last 48h" relativeFrom = "now-48h/h" relativeTo = "now" - -[[transform.investigate.providers]] -field = "event.kind" -excluded = false -queryType = "phrase" -value = "signal" -valueType = "string" - -[[transform.investigate.providers]] -field = "host.name" -excluded = false -queryType = "phrase" -value = "{{host.name}}" -valueType = "string" +providers = [ + [ + {field = "event.kind", excluded = false, queryType = "phrase", value = "signal", valueType = "string"}, + {field = "host.name", excluded = false, queryType = "phrase", value = "{{host.name}}", valueType = "string"} + ] +] [[transform.investigate]] label = "Investigate the Subject Process Network Events" - -[[transform.investigate.providers]] -field = "process.entity_id" -excluded = false -queryType = "phrase" -value = "{{process.entity_id}}" -valueType = "string" - -[[transform.investigate.providers]] -field = "event.category" -excluded = false -queryType = "phrase" -value = "network" -valueType = "string" +providers = [ + [ + {field = "process.entity_id", excluded = false, queryType = "phrase", value = "{{process.entity_id}}", valueType = "string"}, + {field = "event.category", excluded = false, queryType = "phrase", value = "network", valueType = "string"} + ] +] [rule] diff --git a/rules/windows/command_and_control_remote_file_copy_desktopimgdownldr.toml b/rules/windows/command_and_control_remote_file_copy_desktopimgdownldr.toml index dafbd878f30..c71d69c29b5 100644 --- a/rules/windows/command_and_control_remote_file_copy_desktopimgdownldr.toml +++ b/rules/windows/command_and_control_remote_file_copy_desktopimgdownldr.toml @@ -35,56 +35,32 @@ authenticode.path JOIN hash ON services.path = hash.path WHERE authenticode.resu label = "Alerts associated with the user in the last 48h" relativeFrom = "now-48h/h" relativeTo = "now" - -[[transform.investigate.providers]] -field = "event.kind" -excluded = false -queryType = "phrase" -value = "signal" -valueType = "string" - -[[transform.investigate.providers]] -field = "user.id" -excluded = false -queryType = "phrase" -value = "{{user.id}}" -valueType = "string" +providers = [ + [ + {field = "event.kind", excluded = false, queryType = "phrase", value = "signal", valueType = "string"}, + {field = "user.id", excluded = false, queryType = "phrase", value = "{{user.id}}", valueType = "string"} + ] +] [[transform.investigate]] label = "Alerts associated with the host in the last 48h" relativeFrom = "now-48h/h" relativeTo = "now" - -[[transform.investigate.providers]] -field = "event.kind" -excluded = false -queryType = "phrase" -value = "signal" -valueType = "string" - -[[transform.investigate.providers]] -field = "host.name" -excluded = false -queryType = "phrase" -value = "{{host.name}}" -valueType = "string" +providers = [ + [ + {field = "event.kind", excluded = false, queryType = "phrase", value = "signal", valueType = "string"}, + {field = "host.name", excluded = false, queryType = "phrase", value = "{{host.name}}", valueType = "string"}, + ] +] [[transform.investigate]] label = "Investigate the Subject Process Network Events" - -[[transform.investigate.providers]] -field = "process.entity_id" -excluded = false -queryType = "phrase" -value = "{{process.entity_id}}" -valueType = "string" - -[[transform.investigate.providers]] -field = "event.category" -excluded = false -queryType = "phrase" -value = "network" -valueType = "string" +providers = [ + [ + {field = "process.entity_id", excluded = false, queryType = "phrase", value = "{{process.entity_id}}", valueType = "string"}, + {field = "event.category", excluded = false, queryType = "phrase", value = "network", valueType = "string"} + ] +] [rule] diff --git a/rules/windows/command_and_control_remote_file_copy_mpcmdrun.toml b/rules/windows/command_and_control_remote_file_copy_mpcmdrun.toml index 11d24a86486..9315508f798 100644 --- a/rules/windows/command_and_control_remote_file_copy_mpcmdrun.toml +++ b/rules/windows/command_and_control_remote_file_copy_mpcmdrun.toml @@ -35,56 +35,32 @@ authenticode.path JOIN hash ON services.path = hash.path WHERE authenticode.resu label = "Alerts associated with the user in the last 48h" relativeFrom = "now-48h/h" relativeTo = "now" - -[[transform.investigate.providers]] -field = "event.kind" -excluded = false -queryType = "phrase" -value = "signal" -valueType = "string" - -[[transform.investigate.providers]] -field = "user.id" -excluded = false -queryType = "phrase" -value = "{{user.id}}" -valueType = "string" +providers = [ + [ + {field = "event.kind", excluded = false, queryType = "phrase", value = "signal", valueType = "string"}, + {field = "user.id", excluded = false, queryType = "phrase", value = "{{user.id}}", valueType = "string"} + ] +] [[transform.investigate]] label = "Alerts associated with the host in the last 48h" relativeFrom = "now-48h/h" relativeTo = "now" - -[[transform.investigate.providers]] -field = "event.kind" -excluded = false -queryType = "phrase" -value = "signal" -valueType = "string" - -[[transform.investigate.providers]] -field = "host.name" -excluded = false -queryType = "phrase" -value = "{{host.name}}" -valueType = "string" +providers = [ + [ + {field = "event.kind", excluded = false, queryType = "phrase", value = "signal", valueType = "string"}, + {field = "host.name", excluded = false, queryType = "phrase", value = "{{host.name}}", valueType = "string"} + ] +] [[transform.investigate]] label = "Investigate the Subject Process Network Events" - -[[transform.investigate.providers]] -field = "process.entity_id" -excluded = false -queryType = "phrase" -value = "{{process.entity_id}}" -valueType = "string" - -[[transform.investigate.providers]] -field = "event.category" -excluded = false -queryType = "phrase" -value = "network" -valueType = "string" +providers = [ + [ + {field = "process.entity_id", excluded = false, queryType = "phrase", value = "{{process.entity_id}}", valueType = "string"}, + {field = "event.category", excluded = false, queryType = "phrase", value = "network", valueType = "string"} + ] +] [rule] diff --git a/rules/windows/command_and_control_remote_file_copy_powershell.toml b/rules/windows/command_and_control_remote_file_copy_powershell.toml index 740ffa2b026..7d6f6074295 100644 --- a/rules/windows/command_and_control_remote_file_copy_powershell.toml +++ b/rules/windows/command_and_control_remote_file_copy_powershell.toml @@ -35,56 +35,32 @@ authenticode.path JOIN hash ON services.path = hash.path WHERE authenticode.resu label = "Alerts associated with the user in the last 48h" relativeFrom = "now-48h/h" relativeTo = "now" - -[[transform.investigate.providers]] -field = "event.kind" -excluded = false -queryType = "phrase" -value = "signal" -valueType = "string" - -[[transform.investigate.providers]] -field = "user.id" -excluded = false -queryType = "phrase" -value = "{{user.id}}" -valueType = "string" +providers = [ + [ + {field = "event.kind", excluded = false, queryType = "phrase", value = "signal", valueType = "string"}, + {field = "user.id", excluded = false, queryType = "phrase", value = "{{user.id}}", valueType = "string"} + ] +] [[transform.investigate]] label = "Alerts associated with the host in the last 48h" relativeFrom = "now-48h/h" relativeTo = "now" - -[[transform.investigate.providers]] -field = "event.kind" -excluded = false -queryType = "phrase" -value = "signal" -valueType = "string" - -[[transform.investigate.providers]] -field = "host.name" -excluded = false -queryType = "phrase" -value = "{{host.name}}" -valueType = "string" +providers = [ + [ + {field = "event.kind", excluded = false, queryType = "phrase", value = "signal", valueType = "string"}, + {field = "host.name", excluded = false, queryType = "phrase", value = "{{host.name}}", valueType = "string"} + ] +] [[transform.investigate]] label = "Investigate the Subject Process Network Events" - -[[transform.investigate.providers]] -field = "process.entity_id" -excluded = false -queryType = "phrase" -value = "{{process.entity_id}}" -valueType = "string" - -[[transform.investigate.providers]] -field = "event.category" -excluded = false -queryType = "phrase" -value = "network" -valueType = "string" +providers = [ + [ + {field = "process.entity_id", excluded = false, queryType = "phrase", value = "{{process.entity_id}}", valueType = "string"}, + {field = "event.category", excluded = false, queryType = "phrase", value = "network", valueType = "string"} + ] +] [rule] diff --git a/rules_building_block/command_and_control_certutil_network_connection.toml b/rules_building_block/command_and_control_certutil_network_connection.toml index d55645b0e8c..4ef4afc104f 100644 --- a/rules_building_block/command_and_control_certutil_network_connection.toml +++ b/rules_building_block/command_and_control_certutil_network_connection.toml @@ -36,57 +36,33 @@ authenticode.path JOIN hash ON services.path = hash.path WHERE authenticode.resu label = "Alerts associated with the user in the last 48h" relativeFrom = "now-48h/h" relativeTo = "now" - - -[[transform.investigate.providers]] -field = "event.kind" -excluded = false -queryType = "phrase" -value = "signal" -valueType = "string" - -[[transform.investigate.providers]] -field = "user.id" -excluded = false -queryType = "phrase" -value = "{{user.id}}" -valueType = "string" +providers = [ + [ + {field = "event.kind", excluded = false, queryType = "phrase", value = "signal", valueType = "string"}, + {field = "user.id", excluded = false, queryType = "phrase", value = "{{user.id}}", valueType = "string"} + ] +] [[transform.investigate]] label = "Alerts associated with the host in the last 48h" relativeFrom = "now-48h/h" relativeTo = "now" - -[[transform.investigate.providers]] -field = "event.kind" -excluded = false -queryType = "phrase" -value = "signal" -valueType = "string" - -[[transform.investigate.providers]] -field = "host.name" -excluded = false -queryType = "phrase" -value = "{{host.name}}" -valueType = "string" +providers = [ + [ + {field = "event.kind", excluded = false, queryType = "phrase", value = "signal", valueType = "string"}, + {field = "host.name", excluded = false, queryType = "phrase", value = "{{host.name}}", valueType = "string"} + ] +] [[transform.investigate]] label = "Investigate the Subject Process Network Events" +providers = [ + [ + {field = "process.entity_id", excluded = false, queryType = "phrase", value = "{{process.entity_id}}", valueType = "string"}, + {field = "event.category", excluded = false, queryType = "phrase", value = "network", valueType = "string"} + ] +] -[[transform.investigate.providers]] -field = "process.entity_id" -excluded = false -queryType = "phrase" -value = "{{process.entity_id}}" -valueType = "string" - -[[transform.investigate.providers]] -field = "event.category" -excluded = false -queryType = "phrase" -value = "network" -valueType = "string" [rule] author = ["Elastic"] From dc852986239552d7cc927c1f5b01d9b701417e7a Mon Sep 17 00:00:00 2001 From: brokensound77 Date: Fri, 8 Dec 2023 10:50:19 -0700 Subject: [PATCH 8/9] update transform unit tests for investigate --- tests/test_all_rules.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/test_all_rules.py b/tests/test_all_rules.py index fda4aa0e422..50d1d1af82d 100644 --- a/tests/test_all_rules.py +++ b/tests/test_all_rules.py @@ -1220,15 +1220,25 @@ def test_note_has_osquery_warning(self): '(https://www.elastic.co/guide/en/security/master/invest-guide-run-osquery.html) ' 'introduced in Elastic Stack version 8.5.0. Older Elastic Stack versions will display ' 'unrendered Markdown in this guide.') + invest_note_pattern = ('> This investigation guide uses the [Investigate Markdown Plugin]' + '(https://www.elastic.co/guide/en/security/master/interactive-investigation-guides.html)' + ' introduced in Elastic Stack version 8.8.0. Older Elastic Stack versions will display ' + 'unrendered Markdown in this guide.') for rule in self.production_rules.rules: if not rule.contents.get('transform'): continue + osquery = rule.contents.transform.get('osquery') if osquery and osquery_note_pattern not in rule.contents.data.note: self.fail(f'{self.rule_str(rule)} Investigation guides using the Osquery Markdown must contain ' f'the following note:\n{osquery_note_pattern}') + investigate = rule.contents.transform.get('investigate') + if investigate and invest_note_pattern not in rule.contents.data.note: + self.fail(f'{self.rule_str(rule)} Investigation guides using the Investigate Markdown must contain ' + f'the following note:\n{invest_note_pattern}') + def test_plugin_placeholders_match_entries(self): """Test that the number of plugin entries match their respective placeholders in note.""" for rule in self.production_rules.rules: @@ -1276,7 +1286,7 @@ def test_if_plugins_explicitly_defined(self): for rule in self.production_rules.rules: note = rule.contents.data.get('note') if note is not None: - results = re.search(r'(!{osquery|!{insight)', note, re.I | re.M) + results = re.search(r'(!{osquery|!{investigate)', note, re.I | re.M) err_msg = f'{self.rule_str(rule)} investigation guide plugin pattern detected! Use Transform' self.assertIsNone(results, err_msg) From dc8313872941144299c7c88a8074a403805cedea Mon Sep 17 00:00:00 2001 From: brokensound77 Date: Fri, 8 Dec 2023 11:46:06 -0700 Subject: [PATCH 9/9] updated docs with transform --- docs/developing.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 docs/developing.md diff --git a/docs/developing.md b/docs/developing.md new file mode 100644 index 00000000000..b7b57ba9c8b --- /dev/null +++ b/docs/developing.md @@ -0,0 +1,38 @@ +# Developing + +Notes for managing and internal development + +## Transforms + +Transforms are data structures within rules which will be integrated into other fields at build +time for rules, meaning they are not directly converted. + +### CLI + +There are some helper commands to assist with converting transforms into the excpected rule TOML format + +- create transform in Kibana +- export it (or copy it) +- run the following commmand and paste them (multiple) +- copy and paste into rule, with minor format changes if needed + +```console +(detection_dev) ➜ detection-rules git:(initial_inv_queries) python -m detection_rules dev transforms guide-plugin-convert + +█▀▀▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄ ▄ █▀▀▄ ▄ ▄ ▄ ▄▄▄ ▄▄▄ +█ █ █▄▄ █ █▄▄ █ █ █ █ █ █▀▄ █ █▄▄▀ █ █ █ █▄▄ █▄▄ +█▄▄▀ █▄▄ █ █▄▄ █▄▄ █ ▄█▄ █▄█ █ ▀▄█ █ ▀▄ █▄▄█ █▄▄ █▄▄ ▄▄█ + +Enter plugin contents []: !{investigate{"label":"Alerts associated with the host in the last 48h","providers":[[{"field":"event.kind","excluded":false,"queryType":"phrase","value":"signal","valueType":"string"},{"field":"host.name","excluded":false,"queryType":"phrase","value":"{{host.name}}","valueType":"string"}]],"relativeFrom":"now-48h/h","relativeTo":"now"}} +[transform] + +[[transform.investigate]] +label = "Alerts associated with the host in the last 48h" +providers = [[{field = "event.kind", excluded = false, queryType = "phrase", value = "signal", valueType = "string"}, {field = "host.name", excluded = false, queryType = "phrase", value = "{{host.name}}", valueType = "string"}]] +relativeFrom = "now-48h/h" +relativeTo = "now" +``` + +Other transform suppoprt can be found under + +`python -m detection-rules dev transforms -h`