[ML] Adds a 30 day model prune window to non-rare Security jobs (#107752

) (#108058) Adds the model_prune_window setting added in elastic/elasticsearch#75741 to all Security jobs that use functions that support model pruning. This means that the models for split field values that are not seen for 30 days will be dropped. If those split field values are subsequently seen again then new models will be created like for completely new entities. The "rare" function does not support model pruning, so jobs that use the "rare" function are not modified. Co-authored-by: David Roberts <[email protected]>
elastic · Aug 10, 2021 · ffec097 · ffec097
1 parent 6b7330c
commit ffec097
Show file tree

Hide file tree

Showing 10 changed files with 20 additions and 10 deletions.
diff --git a/.../server/models/data_recognizer/modules/security_auth/ml/auth_high_count_logon_events.json b/.../server/models/data_recognizer/modules/security_auth/ml/auth_high_count_logon_events.json
@@ -14,7 +14,8 @@
         "detector_index": 0
       }
     ],
-    "influencers": []
+    "influencers": [],
+    "model_prune_window": "30d"
   },
   "allow_lazy_open": true,
   "analysis_limits": {

diff --git a/...ata_recognizer/modules/security_auth/ml/auth_high_count_logon_events_for_a_source_ip.json b/...ata_recognizer/modules/security_auth/ml/auth_high_count_logon_events_for_a_source_ip.json
@@ -19,7 +19,8 @@
       "source.ip",
       "winlog.event_data.LogonType",
       "user.name"
-    ]
+    ],
+    "model_prune_window": "30d"
   },
   "allow_lazy_open": true,
   "analysis_limits": {

diff --git a/...l/server/models/data_recognizer/modules/security_auth/ml/auth_high_count_logon_fails.json b/...l/server/models/data_recognizer/modules/security_auth/ml/auth_high_count_logon_fails.json
@@ -14,7 +14,8 @@
         "detector_index": 0
       }
     ],
-    "influencers": []
+    "influencers": [],
+    "model_prune_window": "30d"
   },
   "allow_lazy_open": true,
   "analysis_limits": {

diff --git a/...models/data_recognizer/modules/security_network/ml/high_count_by_destination_country.json b/...models/data_recognizer/modules/security_network/ml/high_count_by_destination_country.json
@@ -20,7 +20,8 @@
       "destination.as.organization.name",
       "source.ip",
       "destination.ip"
-    ]
+    ],
+    "model_prune_window": "30d"
   },
   "allow_lazy_open": true,
   "analysis_limits": {

diff --git a/.../server/models/data_recognizer/modules/security_network/ml/high_count_network_denies.json b/.../server/models/data_recognizer/modules/security_network/ml/high_count_network_denies.json
@@ -19,7 +19,8 @@
       "destination.as.organization.name",
       "source.ip",
       "destination.port"
-    ]
+    ],
+    "model_prune_window": "30d"
   },
   "allow_lazy_open": true,
   "analysis_limits": {

diff --git a/.../server/models/data_recognizer/modules/security_network/ml/high_count_network_events.json b/.../server/models/data_recognizer/modules/security_network/ml/high_count_network_events.json
@@ -19,7 +19,8 @@
       "destination.as.organization.name",
       "source.ip",
       "destination.ip"
-    ]
+    ],
+    "model_prune_window": "30d"
   },
   "allow_lazy_open": true,
   "analysis_limits": {

diff --git a/.../models/data_recognizer/modules/siem_auditbeat_auth/ml/suspicious_login_activity_ecs.json b/.../models/data_recognizer/modules/siem_auditbeat_auth/ml/suspicious_login_activity_ecs.json
@@ -19,7 +19,8 @@
       "host.name",
       "user.name",
       "source.ip"
-    ]
+    ],
+    "model_prune_window": "30d"
   },
   "allow_lazy_open": true,
   "analysis_limits": {

diff --git a/.../models/data_recognizer/modules/siem_cloudtrail/ml/high_distinct_count_error_message.json b/.../models/data_recognizer/modules/siem_cloudtrail/ml/high_distinct_count_error_message.json
@@ -18,7 +18,8 @@
       "aws.cloudtrail.user_identity.arn",
       "source.ip",
       "source.geo.city_name"
-    ]
+    ],
+    "model_prune_window": "30d"
   },
   "allow_lazy_open": true,
   "analysis_limits": {

diff --git a/...ml/server/models/data_recognizer/modules/siem_packetbeat/ml/packetbeat_dns_tunneling.json b/...ml/server/models/data_recognizer/modules/siem_packetbeat/ml/packetbeat_dns_tunneling.json
@@ -34,7 +34,8 @@
       "destination.ip",
       "host.name",
       "dns.question.etld_plus_one"
-    ]
+    ],
+    "model_prune_window": "30d"
   },
   "allow_lazy_open": true,
   "analysis_limits": {

diff --git a/...ml/server/models/data_recognizer/modules/siem_winlogbeat/ml/windows_anomalous_script.json b/...ml/server/models/data_recognizer/modules/siem_winlogbeat/ml/windows_anomalous_script.json
@@ -19,7 +19,8 @@
       "host.name",
       "user.name",
       "winlog.event_data.Path"
-    ]
+    ],
+    "model_prune_window": "30d"
   },
   "allow_lazy_open": true,
   "analysis_limits": {