detect: Ensure app-layer decode events are logged

doc/userguide/output/eve/eve-json-format.rst

@@ -173,7 +173,10 @@ The following field is included when "type" has the value "applayer":
 
 * "layer" Indicates the handling layer that detected the event. This will be
   "proto_parser" (protocol parser), "proto_detect" (protocol detection) or
-  "parser."
+  "decoder_detect" (decoder events) or "parser."
+
+"decoder_detect" events are conditions that occur while decoding packet contents
+such as SWF file handling on HTTP payloads.
 
 When ``packethdr`` is enabled, the first 32 bytes of the packet are included
 as a byte64-encoded blob in the main part of record. This applies to events
@@ -260,6 +263,24 @@ Examples
       }
     }
 
+    {
+      "timestamp": "2021-11-29T09:56:36.348743-0500",
+       "flow_id": 1266018553187300,
+       "pcap_cnt": 22799597,
+       "event_type": "anomaly",
+       "src_ip": "127.0.0.1",
+       "src_port": 8080,
+       "dest_ip": "127.0.0.1",
+       "dest_port": 50608,
+       "proto": "TCP",
+       "anomaly": {
+         "app_proto": "http",
+         "type": "applayer",
+         "event": "FILE_DECODER_EVENT_INVALID_SWF_VERSION",
+         "layer": "decoder_detect"
+       }
+    }
+
 Event type: HTTP
 ----------------
 

src/app-layer-events.c

@@ -31,21 +31,29 @@
 
 /* events raised during protocol detection are stored in the
  * packets storage, not in the flow. */
-SCEnumCharMap app_layer_event_pkt_table[ ] = {
-    { "APPLAYER_MISMATCH_PROTOCOL_BOTH_DIRECTIONS",
-      APPLAYER_MISMATCH_PROTOCOL_BOTH_DIRECTIONS },
-    { "APPLAYER_WRONG_DIRECTION_FIRST_DATA",
-      APPLAYER_WRONG_DIRECTION_FIRST_DATA },
-    { "APPLAYER_DETECT_PROTOCOL_ONLY_ONE_DIRECTION",
-      APPLAYER_DETECT_PROTOCOL_ONLY_ONE_DIRECTION },
-    { "APPLAYER_PROTO_DETECTION_SKIPPED",
-      APPLAYER_PROTO_DETECTION_SKIPPED },
-    { "APPLAYER_NO_TLS_AFTER_STARTTLS",
-      APPLAYER_NO_TLS_AFTER_STARTTLS },
-    { "APPLAYER_UNEXPECTED_PROTOCOL",
-      APPLAYER_UNEXPECTED_PROTOCOL },
-    { NULL,
-      -1 },
+SCEnumCharMap app_layer_event_pkt_table[] = {
+    { "APPLAYER_MISMATCH_PROTOCOL_BOTH_DIRECTIONS", APPLAYER_MISMATCH_PROTOCOL_BOTH_DIRECTIONS },
+    { "APPLAYER_WRONG_DIRECTION_FIRST_DATA", APPLAYER_WRONG_DIRECTION_FIRST_DATA },
+    { "APPLAYER_DETECT_PROTOCOL_ONLY_ONE_DIRECTION", APPLAYER_DETECT_PROTOCOL_ONLY_ONE_DIRECTION },
+    { "APPLAYER_PROTO_DETECTION_SKIPPED", APPLAYER_PROTO_DETECTION_SKIPPED },
+    { "APPLAYER_NO_TLS_AFTER_STARTTLS", APPLAYER_NO_TLS_AFTER_STARTTLS },
+    { "APPLAYER_UNEXPECTED_PROTOCOL", APPLAYER_UNEXPECTED_PROTOCOL },
+    { "FILE_DECODER_EVENT_NO_MEM", FILE_DECODER_EVENT_NO_MEM },
+    { "FILE_DECODER_EVENT_INVALID_SWF_LENGTH", FILE_DECODER_EVENT_INVALID_SWF_LENGTH },
+    { "FILE_DECODER_EVENT_INVALID_SWF_VERSION", FILE_DECODER_EVENT_INVALID_SWF_VERSION },
+    { "FILE_DECODER_EVENT_Z_DATA_ERROR", FILE_DECODER_EVENT_Z_DATA_ERROR },
+    { "FILE_DECODER_EVENT_Z_STREAM_ERROR", FILE_DECODER_EVENT_Z_STREAM_ERROR },
+    { "FILE_DECODER_EVENT_Z_BUF_ERROR", FILE_DECODER_EVENT_Z_BUF_ERROR },
+    { "FILE_DECODER_EVENT_Z_UNKNOWN_ERROR", FILE_DECODER_EVENT_Z_UNKNOWN_ERROR },
+    { "FILE_DECODER_EVENT_LZMA_DECODER_ERROR", FILE_DECODER_EVENT_LZMA_DECODER_ERROR },
+    { "FILE_DECODER_EVENT_LZMA_MEMLIMIT_ERROR", FILE_DECODER_EVENT_LZMA_MEMLIMIT_ERROR },
+    { "FILE_DECODER_EVENT_LZMA_OPTIONS_ERROR", FILE_DECODER_EVENT_LZMA_OPTIONS_ERROR },
+    { "FILE_DECODER_EVENT_LZMA_FORMAT_ERROR", FILE_DECODER_EVENT_LZMA_FORMAT_ERROR },
+    { "FILE_DECODER_EVENT_LZMA_DATA_ERROR", FILE_DECODER_EVENT_LZMA_DATA_ERROR },
+    { "FILE_DECODER_EVENT_LZMA_BUF_ERROR", FILE_DECODER_EVENT_LZMA_BUF_ERROR },
+    { "FILE_DECODER_EVENT_LZMA_UNKNOWN_ERROR", FILE_DECODER_EVENT_LZMA_UNKNOWN_ERROR },
+    { "DETECT_EVENT_TOO_MANY_BUFFERS", DETECT_EVENT_TOO_MANY_BUFFERS },
+    { NULL, -1 },
 };
 
 int AppLayerGetEventInfoById(int event_id, const char **event_name,

src/app-layer-events.h

@@ -50,6 +50,25 @@ enum {
     APPLAYER_PROTO_DETECTION_SKIPPED,
     APPLAYER_NO_TLS_AFTER_STARTTLS,
     APPLAYER_UNEXPECTED_PROTOCOL,
+#ifdef UNITTESTS
+    DET_CTX_EVENT_TEST,
+#endif
+    FILE_DECODER_EVENT_NO_MEM,
+    FILE_DECODER_EVENT_INVALID_SWF_LENGTH,
+    FILE_DECODER_EVENT_INVALID_SWF_VERSION,
+    FILE_DECODER_EVENT_Z_DATA_ERROR,
+    FILE_DECODER_EVENT_Z_STREAM_ERROR,
+    FILE_DECODER_EVENT_Z_BUF_ERROR,
+    FILE_DECODER_EVENT_Z_UNKNOWN_ERROR,
+    FILE_DECODER_EVENT_LZMA_DECODER_ERROR,
+    FILE_DECODER_EVENT_LZMA_MEMLIMIT_ERROR,
+    FILE_DECODER_EVENT_LZMA_OPTIONS_ERROR,
+    FILE_DECODER_EVENT_LZMA_FORMAT_ERROR,
+    FILE_DECODER_EVENT_LZMA_DATA_ERROR,
+    FILE_DECODER_EVENT_LZMA_BUF_ERROR,
+    FILE_DECODER_EVENT_LZMA_UNKNOWN_ERROR,
+
+    DETECT_EVENT_TOO_MANY_BUFFERS,
 };
 
 /* the event types for app events */

src/decode.h

@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2020 Open Information Security Foundation
+/* Copyright (C) 2007-2021 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -577,6 +577,8 @@ typedef struct Packet_
 
     AppLayerDecoderEvents *app_layer_events;
 
+    AppLayerDecoderEvents *decoder_events;
+
     /* double linked list ptrs */
     struct Packet_ *next;
     struct Packet_ *prev;
@@ -829,6 +831,7 @@ void CaptureStatsSetup(ThreadVars *tv, CaptureStats *s);
         (p)->tunnel_tpr_cnt = 0;                                                                   \
         (p)->events.cnt = 0;                                                                       \
         AppLayerDecoderEventsResetEvents((p)->app_layer_events);                                   \
+        AppLayerDecoderEventsResetEvents((p)->decoder_events);                                     \
         (p)->next = NULL;                                                                          \
         (p)->prev = NULL;                                                                          \
         (p)->root = NULL;                                                                          \
@@ -847,17 +850,18 @@ void CaptureStatsSetup(ThreadVars *tv, CaptureStats *s);
 /**
  *  \brief Cleanup a packet so that we can free it. No memset needed..
  */
-#define PACKET_DESTRUCTOR(p) do {                  \
-        if ((p)->pktvar != NULL) {              \
-            PktVarFree((p)->pktvar);            \
-        }                                       \
-        PACKET_FREE_EXTDATA((p));               \
-        SCMutexDestroy(&(p)->tunnel_mutex);     \
-        AppLayerDecoderEventsFreeEvents(&(p)->app_layer_events); \
-        PACKET_PROFILING_RESET((p));            \
+#define PACKET_DESTRUCTOR(p)                                                                       \
+    do {                                                                                           \
+        if ((p)->pktvar != NULL) {                                                                 \
+            PktVarFree((p)->pktvar);                                                               \
+        }                                                                                          \
+        PACKET_FREE_EXTDATA((p));                                                                  \
+        SCMutexDestroy(&(p)->tunnel_mutex);                                                        \
+        AppLayerDecoderEventsFreeEvents(&(p)->app_layer_events);                                   \
+        AppLayerDecoderEventsFreeEvents(&(p)->decoder_events);                                     \
+        PACKET_PROFILING_RESET((p));                                                               \
     } while (0)
 
-
 /* macro's for setting the action
  * handle the case of a root packet
  * for tunnels */

src/detect-engine-build.c

@@ -408,7 +408,9 @@ PacketCreateMask(Packet *p, SignatureMask *mask, AppProto alproto,
         (*mask) |= SIG_MASK_REQUIRE_NO_PAYLOAD;
     }
 
-    if (p->events.cnt > 0 || app_decoder_events != 0 || p->app_layer_events != NULL) {
+    if (p->events.cnt > 0 || app_decoder_events ||
+            (p->app_layer_events != NULL && p->app_layer_events->cnt) ||
+            (p->decoder_events != NULL && p->decoder_events->cnt)) {
         SCLogDebug("packet/flow has events set");
         (*mask) |= SIG_MASK_REQUIRE_ENGINE_EVENT;
     }

src/detect-engine.c

@@ -3032,8 +3032,6 @@ static void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx)
 #endif
     }
 
-    AppLayerDecoderEventsFreeEvents(&det_ctx->decoder_events);
-
     SCFree(det_ctx);
 }
 
@@ -4300,15 +4298,10 @@ const char *DetectSigmatchListEnumToString(enum DetectSigmatchListEnum type)
 /* events api */
 void DetectEngineSetEvent(DetectEngineThreadCtx *det_ctx, uint8_t e)
 {
-    AppLayerDecoderEventsSetEventRaw(&det_ctx->decoder_events, e);
+    AppLayerDecoderEventsSetEventRaw(&det_ctx->p->decoder_events, e);
     det_ctx->events++;
 }
 
-AppLayerDecoderEvents *DetectEngineGetEvents(DetectEngineThreadCtx *det_ctx)
-{
-    return det_ctx->decoder_events;
-}
-
 int DetectEngineGetEventInfo(const char *event_name, int *event_id,
                              AppLayerEventType *event_type)
 {

src/detect.c

@@ -888,6 +888,9 @@ static DetectRunScratchpad DetectRunSetup(
         app_decoder_events = AppLayerParserHasDecoderEvents(pflow->alparser);
     }
 
+    /* Invariant during detect stage */
+    det_ctx->p = p;
+
     DetectRunScratchpad pad = { alproto, flow_flags, app_decoder_events, NULL, 0 };
     PACKET_PROFILING_DETECT_END(p, PROF_DETECT_SETUP);
     return pad;
@@ -934,6 +937,9 @@ static void DetectRunCleanup(DetectEngineThreadCtx *det_ctx,
                     det_ctx->raw_stream_progress);
         }
     }
+
+    det_ctx->p = NULL;
+
     PACKET_PROFILING_DETECT_END(p, PROF_DETECT_CLEANUP);
     SCReturn;
 }
@@ -1512,8 +1518,9 @@ static void DetectRunTx(ThreadVars *tv,
 next:
         InspectionBufferClean(det_ctx);
 
-        if (!ires.has_next)
+        if (!ires.has_next) {
             break;
+        }
     }
 }
 

src/detect.h

@@ -1142,7 +1142,6 @@ typedef struct DetectEngineThreadCtx_ {
     int base64_decoded_len;
     int base64_decoded_len_max;
 
-    AppLayerDecoderEvents *decoder_events;
     uint16_t events;
 
 #ifdef DEBUG
@@ -1212,29 +1211,6 @@ typedef struct SigTableElmt_ {
 
 } SigTableElmt;
 
-/* event code */
-enum {
-#ifdef UNITTESTS
-    DET_CTX_EVENT_TEST,
-#endif
-    FILE_DECODER_EVENT_NO_MEM,
-    FILE_DECODER_EVENT_INVALID_SWF_LENGTH,
-    FILE_DECODER_EVENT_INVALID_SWF_VERSION,
-    FILE_DECODER_EVENT_Z_DATA_ERROR,
-    FILE_DECODER_EVENT_Z_STREAM_ERROR,
-    FILE_DECODER_EVENT_Z_BUF_ERROR,
-    FILE_DECODER_EVENT_Z_UNKNOWN_ERROR,
-    FILE_DECODER_EVENT_LZMA_DECODER_ERROR,
-    FILE_DECODER_EVENT_LZMA_MEMLIMIT_ERROR,
-    FILE_DECODER_EVENT_LZMA_OPTIONS_ERROR,
-    FILE_DECODER_EVENT_LZMA_FORMAT_ERROR,
-    FILE_DECODER_EVENT_LZMA_DATA_ERROR,
-    FILE_DECODER_EVENT_LZMA_BUF_ERROR,
-    FILE_DECODER_EVENT_LZMA_UNKNOWN_ERROR,
-
-    DETECT_EVENT_TOO_MANY_BUFFERS,
-};
-
 #define SIG_GROUP_HEAD_HAVERAWSTREAM    BIT_U32(0)
 #ifdef HAVE_MAGIC
 #define SIG_GROUP_HEAD_HAVEFILEMAGIC    BIT_U32(20)
@@ -1501,7 +1477,6 @@ void DetectMetadataHashFree(DetectEngineCtx *de_ctx);
 
 /* events */
 void DetectEngineSetEvent(DetectEngineThreadCtx *det_ctx, uint8_t e);
-AppLayerDecoderEvents *DetectEngineGetEvents(DetectEngineThreadCtx *det_ctx);
 int DetectEngineGetEventInfo(const char *event_name, int *event_id,
                              AppLayerEventType *event_type);
 

src/output-json-anomaly.c

@@ -242,6 +242,11 @@ static inline bool AnomalyHasPacketAppLayerEvents(const Packet *p)
     return p->app_layer_events && p->app_layer_events->cnt;
 }
 
+static inline bool AnomalyHasPacketDecoderEvents(const Packet *p)
+{
+    return p->decoder_events && p->decoder_events->cnt;
+}
+
 static int AnomalyJson(ThreadVars *tv, JsonAnomalyLogThread *aft, const Packet *p)
 {
     int rc = TM_ECODE_OK;
@@ -261,6 +266,11 @@ static int AnomalyJson(ThreadVars *tv, JsonAnomalyLogThread *aft, const Packet *
                                                  true, "proto_detect", TX_ID_UNUSED);
         }
 
+        if (rc == TM_ECODE_OK && AnomalyHasPacketDecoderEvents(p)) {
+            rc = AnomalyAppLayerDecoderEventJson(
+                    aft, p, p->decoder_events, true, "decoder_detect", TX_ID_UNUSED);
+        }
+
         /* parser state events */
         if (rc == TM_ECODE_OK && AnomalyHasParserEvents(p)) {
             SCLogDebug("Checking for anomaly events; alproto %d", p->flow->alproto);
@@ -283,9 +293,8 @@ static int JsonAnomalyLogger(ThreadVars *tv, void *thread_data, const Packet *p)
 
 static int JsonAnomalyLogCondition(ThreadVars *tv, const Packet *p)
 {
-    return p->events.cnt > 0 ||
-           (p->app_layer_events && p->app_layer_events->cnt > 0) ||
-           AnomalyHasParserEvents(p);
+    return p->events.cnt > 0 || (p->app_layer_events && p->app_layer_events->cnt > 0) ||
+           (p->decoder_events && p->decoder_events->cnt > 0) || AnomalyHasParserEvents(p);
 }
 
 static TmEcode JsonAnomalyLogThreadInit(ThreadVars *t, const void *initdata, void **data)

src/util-file-decompression.c

@@ -19,7 +19,7 @@
  *
  * \author Giuseppe Longo <[email protected]>
  *
- * \brief Decompress files transfered via HTTP corresponding to file_data
+ * \brief Decompress files transferred via HTTP corresponding to file_data
  * keyword.
  *
  */
@@ -60,13 +60,13 @@ int FileIsSwfFile(const uint8_t *buffer, uint32_t buffer_len)
  * \param buffer compressed buffer
  * \param buffer_len compressed buffer length
  * \param decompressed_buffer buffer that store decompressed data
- * \param decompressed_buffer_len decompressesd data length
+ * \param decompressed_buffer_len decompressed data length
  * \param swf_type decompression algorithm to use
  * \param decompress_depth how much decompressed data we want to store
  * \param compress_depth how much compressed data we want to decompress
  *
  * \retval 1 if decompression works
- * \retval 0 an error occured, and event set
+ * \retval 0 an error occurred, and event set
  */
 int FileSwfDecompression(const uint8_t *buffer, uint32_t buffer_len,
                          DetectEngineThreadCtx *det_ctx,