Skip to content

Commit 7d7b28b

Browse files
w0rk3rgithub-actions[bot]
authored andcommitted
[Security Content] Introduce Investigate Plugin in Investigation Guides (#3080)
* [Security Content] Introduce Investigate Plugin in Investigation Guides * Add compatibility note * Update Transform format * update transform unit tests for investigate * updated docs with transform --------- Co-authored-by: brokensound77 <[email protected]> Co-authored-by: Justin Ibarra <[email protected]> (cherry picked from commit aeb1f91)
1 parent 361b773 commit 7d7b28b

9 files changed

+247
-26
lines changed

detection_rules/rule.py

+12-16
Original file line numberDiff line numberDiff line change
@@ -91,34 +91,30 @@ class OsQuery:
9191
ecs_mapping: Optional[Dict[str, Dict[Literal['field', 'value'], str]]]
9292

9393
@dataclass(frozen=True)
94-
class Insight:
94+
class Investigate:
9595
@dataclass(frozen=True)
9696
class Provider:
97+
excluded: bool
9798
field: str
99+
queryType: definitions.InvestigateProviderQueryType
98100
value: str
99-
type: str
101+
valueType: definitions.InvestigateProviderValueType
100102

101103
label: str
104+
description: Optional[str]
102105
providers: List[List[Provider]]
106+
relativeFrom: Optional[str]
107+
relativeTo: Optional[str]
103108

104109
# these must be lists in order to have more than one. Their index in the list is how they will be referenced in the
105110
# note string templates
106111
osquery: Optional[List[OsQuery]]
107-
insight: Optional[List[Insight]]
112+
investigate: Optional[List[Investigate]]
108113

109-
@validates_schema
110-
def validate_transforms(self, value: dict, **kwargs):
111-
"""Validate transform fields."""
112-
# temporarily invalidate insights until schema stabilizes
113-
insight = value.get('insight')
114-
if insight is not None:
115-
raise NotImplementedError('Insights are not stable yet.')
116-
return
117-
118-
def render_insight_osquery_to_string(self) -> Dict[Literal['osquery', 'insight'], List[str]]:
114+
def render_investigate_osquery_to_string(self) -> Dict[definitions.TransformTypes, List[str]]:
119115
obj = self.to_dict()
120116

121-
rendered: Dict[Literal['osquery', 'insight'], List[str]] = {'osquery': [], 'insight': []}
117+
rendered: Dict[definitions.TransformTypes, List[str]] = {'osquery': [], 'investigate': []}
122118
for plugin, entries in obj.items():
123119
for entry in entries:
124120
rendered[plugin].append(f'!{{{plugin}{json.dumps(entry, sort_keys=True, separators=(",", ":"))}}}')
@@ -343,12 +339,12 @@ def process_transforms(cls, transform: RuleTransform, obj: dict) -> dict:
343339
# only create functions that CAREFULLY mutate the obj dict
344340

345341
def process_note_plugins():
346-
"""Format the note field with osquery and insight plugin strings."""
342+
"""Format the note field with osquery and investigate plugin strings."""
347343
note = obj.get('note')
348344
if not note:
349345
return
350346

351-
rendered = transform.render_insight_osquery_to_string()
347+
rendered = transform.render_investigate_osquery_to_string()
352348
rendered_patterns = {}
353349
for plugin, entries in rendered.items():
354350
rendered_patterns.update(**{f'{plugin}_{i}': e for i, e in enumerate(entries)})

detection_rules/schemas/definitions.py

+3
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,8 @@
140140
Date = NewType('Date', str, validate=validate.Regexp(DATE_PATTERN))
141141
FilterLanguages = Literal["kuery", "lucene", "eql", "esql"]
142142
Interval = NewType('Interval', str, validate=validate.Regexp(INTERVAL_PATTERN))
143+
InvestigateProviderQueryType = Literal["phrase", "range"]
144+
InvestigateProviderValueType = Literal["string", "boolean"]
143145
Markdown = NewType("MarkdownField", CodeString)
144146
Maturity = Literal['development', 'experimental', 'beta', 'production', 'deprecated']
145147
MaxSignals = NewType("MaxSignals", int, validate=validate.Range(min=1))
@@ -160,6 +162,7 @@
160162
ThresholdValue = NewType("ThresholdValue", int, validate=validate.Range(min=1))
161163
TimelineTemplateId = NewType('TimelineTemplateId', str, validate=validate.OneOf(list(TIMELINE_TEMPLATES)))
162164
TimelineTemplateTitle = NewType('TimelineTemplateTitle', str, validate=validate.OneOf(TIMELINE_TEMPLATES.values()))
165+
TransformTypes = Literal["osquery", "investigate"]
163166
UUIDString = NewType('UUIDString', str, validate=validate.Regexp(UUID_PATTERN))
164167
BuildingBlockType = Literal['default']
165168

docs/developing.md

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# Developing
2+
3+
Notes for managing and internal development
4+
5+
## Transforms
6+
7+
Transforms are data structures within rules which will be integrated into other fields at build
8+
time for rules, meaning they are not directly converted.
9+
10+
### CLI
11+
12+
There are some helper commands to assist with converting transforms into the excpected rule TOML format
13+
14+
- create transform in Kibana
15+
- export it (or copy it)
16+
- run the following commmand and paste them (multiple)
17+
- copy and paste into rule, with minor format changes if needed
18+
19+
```console
20+
(detection_dev) ➜ detection-rules git:(initial_inv_queries) python -m detection_rules dev transforms guide-plugin-convert
21+
22+
█▀▀▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄ ▄ █▀▀▄ ▄ ▄ ▄ ▄▄▄ ▄▄▄
23+
█ █ █▄▄ █ █▄▄ █ █ █ █ █ █▀▄ █ █▄▄▀ █ █ █ █▄▄ █▄▄
24+
█▄▄▀ █▄▄ █ █▄▄ █▄▄ █ ▄█▄ █▄█ █ ▀▄█ █ ▀▄ █▄▄█ █▄▄ █▄▄ ▄▄█
25+
26+
Enter plugin contents []: !{investigate{"label":"Alerts associated with the host in the last 48h","providers":[[{"field":"event.kind","excluded":false,"queryType":"phrase","value":"signal","valueType":"string"},{"field":"host.name","excluded":false,"queryType":"phrase","value":"{{host.name}}","valueType":"string"}]],"relativeFrom":"now-48h/h","relativeTo":"now"}}
27+
[transform]
28+
29+
[[transform.investigate]]
30+
label = "Alerts associated with the host in the last 48h"
31+
providers = [[{field = "event.kind", excluded = false, queryType = "phrase", value = "signal", valueType = "string"}, {field = "host.name", excluded = false, queryType = "phrase", value = "{{host.name}}", valueType = "string"}]]
32+
relativeFrom = "now-48h/h"
33+
relativeTo = "now"
34+
```
35+
36+
Other transform suppoprt can be found under
37+
38+
`python -m detection-rules dev transforms -h`

rules/windows/command_and_control_common_webservices.toml

+38-3
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ integration = ["endpoint"]
44
maturity = "production"
55
min_stack_comments = "New fields added: required_fields, related_integrations, setup"
66
min_stack_version = "8.3.0"
7-
updated_date = "2023/10/16"
7+
updated_date = "2023/12/07"
88

99
[transform]
1010
[[transform.osquery]]
@@ -31,6 +31,38 @@ services.path FROM services JOIN authenticode ON services.path = authenticode.pa
3131
authenticode.path JOIN hash ON services.path = hash.path WHERE authenticode.result != 'trusted'
3232
"""
3333

34+
[[transform.investigate]]
35+
label = "Alerts associated with the user in the last 48h"
36+
relativeFrom = "now-48h/h"
37+
relativeTo = "now"
38+
providers = [
39+
[
40+
{field = "event.kind", excluded = false, queryType = "phrase", value = "signal", valueType = "string"},
41+
{field = "user.id", excluded = false, queryType = "phrase", value = "{{user.id}}", valueType = "string"}
42+
]
43+
]
44+
45+
46+
[[transform.investigate]]
47+
label = "Alerts associated with the host in the last 48h"
48+
relativeFrom = "now-48h/h"
49+
relativeTo = "now"
50+
providers = [
51+
[
52+
{field = "event.kind", excluded = false, queryType = "phrase", value = "signal", valueType = "string"},
53+
{field = "host.name", excluded = false, queryType = "phrase", value = "{{host.name}}", valueType = "string"}
54+
]
55+
]
56+
57+
[[transform.investigate]]
58+
label = "Investigate the Subject Process Network Events"
59+
providers = [
60+
[
61+
{field = "process.entity_id", excluded = false, queryType = "phrase", value = "{{process.entity_id}}", valueType = "string"},
62+
{field = "event.category", excluded = false, queryType = "phrase", value = "network", valueType = "string"}
63+
]
64+
]
65+
3466

3567
[rule]
3668
author = ["Elastic"]
@@ -55,18 +87,22 @@ This rule looks for processes outside known legitimate program locations communi
5587
5688
> **Note**:
5789
> This investigation guide uses the [Osquery Markdown Plugin](https://www.elastic.co/guide/en/security/master/invest-guide-run-osquery.html) introduced in Elastic Stack version 8.5.0. Older Elastic Stack versions will display unrendered Markdown in this guide.
90+
> This investigation guide uses the [Investigate Markdown Plugin](https://www.elastic.co/guide/en/security/master/interactive-investigation-guides.html) introduced in Elastic Stack version 8.8.0. Older Elastic Stack versions will display unrendered Markdown in this guide.
5891
5992
#### Possible investigation steps
6093
61-
- Investigate the script execution chain (parent process tree) for unknown processes. Examine their executable files for prevalence, whether they are located in expected locations, and if they are signed with valid digital signatures.
94+
- Investigate the process execution chain (parent process tree) for unknown processes. Examine their executable files for prevalence, whether they are located in expected locations, and if they are signed with valid digital signatures.
6295
- Investigate other alerts associated with the user/host during the past 48 hours.
96+
- $investigate_0
97+
- $investigate_1
6398
- Verify whether the digital signature exists in the executable.
6499
- Identify the operation type (upload, download, tunneling, etc.).
65100
- Examine the host for derived artifacts that indicate suspicious activities:
66101
- Analyze the process executable using a private sandboxed analysis system.
67102
- Observe and collect information about the following activities in both the sandbox and the alert subject host:
68103
- Attempts to contact external domains and addresses.
69104
- Use the Elastic Defend network events to determine domains and addresses contacted by the subject process by filtering by the process' `process.entity_id`.
105+
- $investigate_2
70106
- Examine the DNS cache for suspicious or anomalous entries.
71107
- $osquery_0
72108
- Use the Elastic Defend registry events to examine registry keys accessed, modified, or created by the related processes in the process tree.
@@ -75,7 +111,6 @@ This rule looks for processes outside known legitimate program locations communi
75111
- $osquery_2
76112
- $osquery_3
77113
- Retrieve the files' SHA-256 hash values using the PowerShell `Get-FileHash` cmdlet and search for the existence and reputation of the hashes in resources like VirusTotal, Hybrid-Analysis, CISCO Talos, Any.run, etc.
78-
- Investigate potentially compromised accounts. Analysts can do this by searching for login events (for example, 4624) to the target host after the registry modification.
79114
80115
### False positive analysis
81116

rules/windows/command_and_control_remote_file_copy_desktopimgdownldr.toml

+36-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
creation_date = "2020/09/03"
33
integration = ["endpoint", "windows"]
44
maturity = "production"
5-
updated_date = "2023/10/23"
5+
updated_date = "2023/12/07"
66
min_stack_comments = "New fields added: required_fields, related_integrations, setup"
77
min_stack_version = "8.3.0"
88

@@ -31,6 +31,37 @@ services.path FROM services JOIN authenticode ON services.path = authenticode.pa
3131
authenticode.path JOIN hash ON services.path = hash.path WHERE authenticode.result != 'trusted'
3232
"""
3333

34+
[[transform.investigate]]
35+
label = "Alerts associated with the user in the last 48h"
36+
relativeFrom = "now-48h/h"
37+
relativeTo = "now"
38+
providers = [
39+
[
40+
{field = "event.kind", excluded = false, queryType = "phrase", value = "signal", valueType = "string"},
41+
{field = "user.id", excluded = false, queryType = "phrase", value = "{{user.id}}", valueType = "string"}
42+
]
43+
]
44+
45+
[[transform.investigate]]
46+
label = "Alerts associated with the host in the last 48h"
47+
relativeFrom = "now-48h/h"
48+
relativeTo = "now"
49+
providers = [
50+
[
51+
{field = "event.kind", excluded = false, queryType = "phrase", value = "signal", valueType = "string"},
52+
{field = "host.name", excluded = false, queryType = "phrase", value = "{{host.name}}", valueType = "string"},
53+
]
54+
]
55+
56+
[[transform.investigate]]
57+
label = "Investigate the Subject Process Network Events"
58+
providers = [
59+
[
60+
{field = "process.entity_id", excluded = false, queryType = "phrase", value = "{{process.entity_id}}", valueType = "string"},
61+
{field = "event.category", excluded = false, queryType = "phrase", value = "network", valueType = "string"}
62+
]
63+
]
64+
3465

3566
[rule]
3667
author = ["Elastic"]
@@ -53,20 +84,24 @@ The `Desktopimgdownldr.exe` utility is used to to configure lockscreen/desktop i
5384
5485
> **Note**:
5586
> This investigation guide uses the [Osquery Markdown Plugin](https://www.elastic.co/guide/en/security/master/invest-guide-run-osquery.html) introduced in Elastic Stack version 8.5.0. Older Elastic Stack versions will display unrendered Markdown in this guide.
87+
> This investigation guide uses the [Investigate Markdown Plugin](https://www.elastic.co/guide/en/security/master/interactive-investigation-guides.html) introduced in Elastic Stack version 8.8.0. Older Elastic Stack versions will display unrendered Markdown in this guide.
5688
5789
#### Possible investigation steps
5890
5991
- Investigate the process execution chain (parent process tree) for unknown processes. Examine their executable files for prevalence, whether they are located in expected locations, and if they are signed with valid digital signatures.
6092
- Identify the user account that performed the action and whether it should perform this kind of action.
6193
- Contact the account owner and confirm whether they are aware of this activity.
6294
- Investigate other alerts associated with the user/host during the past 48 hours.
95+
- $investigate_0
96+
- $investigate_1
6397
- Assess whether this behavior is prevalent in the environment by looking for similar occurrences across hosts.
6498
- Check the reputation of the domain or IP address used to host the downloaded file or if the user downloaded the file from an internal system.
6599
- Examine the host for derived artifacts that indicate suspicious activities:
66100
- Analyze the file using a private sandboxed analysis system.
67101
- Observe and collect information about the following activities in both the sandbox and the alert subject host:
68102
- Attempts to contact external domains and addresses.
69103
- Use the Elastic Defend network events to determine domains and addresses contacted by the subject process by filtering by the process' `process.entity_id`.
104+
- $investigate_2
70105
- Examine the DNS cache for suspicious or anomalous entries.
71106
- $osquery_0
72107
- Use the Elastic Defend registry events to examine registry keys accessed, modified, or created by the related processes in the process tree.

rules/windows/command_and_control_remote_file_copy_mpcmdrun.toml

+36-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
creation_date = "2020/09/03"
33
integration = ["endpoint", "windows"]
44
maturity = "production"
5-
updated_date = "2023/10/23"
5+
updated_date = "2023/12/07"
66
min_stack_comments = "New fields added: required_fields, related_integrations, setup"
77
min_stack_version = "8.3.0"
88

@@ -31,6 +31,37 @@ services.path FROM services JOIN authenticode ON services.path = authenticode.pa
3131
authenticode.path JOIN hash ON services.path = hash.path WHERE authenticode.result != 'trusted'
3232
"""
3333

34+
[[transform.investigate]]
35+
label = "Alerts associated with the user in the last 48h"
36+
relativeFrom = "now-48h/h"
37+
relativeTo = "now"
38+
providers = [
39+
[
40+
{field = "event.kind", excluded = false, queryType = "phrase", value = "signal", valueType = "string"},
41+
{field = "user.id", excluded = false, queryType = "phrase", value = "{{user.id}}", valueType = "string"}
42+
]
43+
]
44+
45+
[[transform.investigate]]
46+
label = "Alerts associated with the host in the last 48h"
47+
relativeFrom = "now-48h/h"
48+
relativeTo = "now"
49+
providers = [
50+
[
51+
{field = "event.kind", excluded = false, queryType = "phrase", value = "signal", valueType = "string"},
52+
{field = "host.name", excluded = false, queryType = "phrase", value = "{{host.name}}", valueType = "string"}
53+
]
54+
]
55+
56+
[[transform.investigate]]
57+
label = "Investigate the Subject Process Network Events"
58+
providers = [
59+
[
60+
{field = "process.entity_id", excluded = false, queryType = "phrase", value = "{{process.entity_id}}", valueType = "string"},
61+
{field = "event.category", excluded = false, queryType = "phrase", value = "network", valueType = "string"}
62+
]
63+
]
64+
3465

3566
[rule]
3667
author = ["Elastic"]
@@ -50,19 +81,23 @@ The `MpCmdRun.exe` is a command-line tool part of Windows Defender and is used t
5081
5182
> **Note**:
5283
> This investigation guide uses the [Osquery Markdown Plugin](https://www.elastic.co/guide/en/security/master/invest-guide-run-osquery.html) introduced in Elastic Stack version 8.5.0. Older Elastic Stack versions will display unrendered Markdown in this guide.
84+
> This investigation guide uses the [Investigate Markdown Plugin](https://www.elastic.co/guide/en/security/master/interactive-investigation-guides.html) introduced in Elastic Stack version 8.8.0. Older Elastic Stack versions will display unrendered Markdown in this guide.
5385
5486
#### Possible investigation steps
5587
5688
- Investigate the process execution chain (parent process tree) for unknown processes. Examine their executable files for prevalence, whether they are located in expected locations, and if they are signed with valid digital signatures.
5789
- Identify the user account that performed the action and whether it should perform this kind of action.
5890
- Contact the account owner and confirm whether they are aware of this activity.
5991
- Investigate other alerts associated with the user/host during the past 48 hours.
92+
- $investigate_0
93+
- $investigate_1
6094
- Check the reputation of the domain or IP address used to host the downloaded file.
6195
- Examine the host for derived artifacts that indicate suspicious activities:
6296
- Analyze the file using a private sandboxed analysis system.
6397
- Observe and collect information about the following activities in both the sandbox and the alert subject host:
6498
- Attempts to contact external domains and addresses.
6599
- Use the Elastic Defend network events to determine domains and addresses contacted by the subject process by filtering by the process' `process.entity_id`.
100+
- $investigate_2
66101
- Examine the DNS cache for suspicious or anomalous entries.
67102
- $osquery_0
68103
- Use the Elastic Defend registry events to examine registry keys accessed, modified, or created by the related processes in the process tree.

0 commit comments

Comments
 (0)