gen/refactor: transform and misc related changes for improved transform support

rust/src/detect/transforms/casechange.rs

@@ -39,7 +39,7 @@ fn tolower_transform_do(input: &[u8], output: &mut [u8]) {
     }
 }
 
-unsafe extern "C" fn tolower_transform(buffer: *mut c_void, _ctx: *mut c_void) {
+unsafe extern "C" fn tolower_transform(_det: *mut c_void, buffer: *mut c_void, _ctx: *mut c_void) {
     let input = InspectionBufferPtr(buffer);
     let input_len = InspectionBufferLength(buffer);
     if input.is_null() || input_len == 0 {
@@ -99,7 +99,7 @@ fn toupper_transform_do(input: &[u8], output: &mut [u8]) {
     }
 }
 
-unsafe extern "C" fn toupper_transform(buffer: *mut c_void, _ctx: *mut c_void) {
+unsafe extern "C" fn toupper_transform(_det: *mut c_void, buffer: *mut c_void, _ctx: *mut c_void) {
     let input = InspectionBufferPtr(buffer);
     let input_len = InspectionBufferLength(buffer);
     if input.is_null() || input_len == 0 {

rust/src/detect/transforms/compress_whitespace.rs

@@ -49,7 +49,7 @@ fn compress_whitespace_transform_do(input: &[u8], output: &mut [u8]) -> u32 {
     return nb as u32;
 }
 
-unsafe extern "C" fn compress_whitespace_transform(buffer: *mut c_void, _ctx: *mut c_void) {
+unsafe extern "C" fn compress_whitespace_transform(_det: *mut c_void, buffer: *mut c_void, _ctx: *mut c_void) {
     let input = InspectionBufferPtr(buffer);
     let input_len = InspectionBufferLength(buffer);
     if input.is_null() || input_len == 0 {

rust/src/detect/transforms/dotprefix.rs

@@ -41,7 +41,7 @@ fn dot_prefix_transform_do(input: &[u8], output: &mut [u8]) {
     output[0] = b'.';
 }
 
-unsafe extern "C" fn dot_prefix_transform(buffer: *mut c_void, _ctx: *mut c_void) {
+unsafe extern "C" fn dot_prefix_transform(_det: *mut c_void, buffer: *mut c_void, _ctx: *mut c_void) {
     let input_len = InspectionBufferLength(buffer);
     if input_len == 0 {
         return;

rust/src/detect/transforms/hash.rs

@@ -49,7 +49,7 @@ fn md5_transform_do(input: &[u8], output: &mut [u8]) {
     Md5::new().chain(input).finalize_into(output.into());
 }
 
-unsafe extern "C" fn md5_transform(buffer: *mut c_void, _ctx: *mut c_void) {
+unsafe extern "C" fn md5_transform(_det: *mut c_void, buffer: *mut c_void, _ctx: *mut c_void) {
     let input = InspectionBufferPtr(buffer);
     let input_len = InspectionBufferLength(buffer);
     if input.is_null() || input_len == 0 {
@@ -101,7 +101,7 @@ fn sha1_transform_do(input: &[u8], output: &mut [u8]) {
     Sha1::new().chain(input).finalize_into(output.into());
 }
 
-unsafe extern "C" fn sha1_transform(buffer: *mut c_void, _ctx: *mut c_void) {
+unsafe extern "C" fn sha1_transform(_det: *mut c_void, buffer: *mut c_void, _ctx: *mut c_void) {
     let input = InspectionBufferPtr(buffer);
     let input_len = InspectionBufferLength(buffer);
     if input.is_null() || input_len == 0 {
@@ -153,7 +153,7 @@ fn sha256_transform_do(input: &[u8], output: &mut [u8]) {
     Sha256::new().chain(input).finalize_into(output.into());
 }
 
-unsafe extern "C" fn sha256_transform(buffer: *mut c_void, _ctx: *mut c_void) {
+unsafe extern "C" fn sha256_transform(_det: *mut c_void, buffer: *mut c_void, _ctx: *mut c_void) {
     let input = InspectionBufferPtr(buffer);
     let input_len = InspectionBufferLength(buffer);
     if input.is_null() || input_len == 0 {

rust/src/detect/transforms/http_headers.rs

@@ -52,7 +52,7 @@ fn header_lowertransform_do(input: &[u8], output: &mut [u8]) {
     }
 }
 
-unsafe extern "C" fn header_lowertransform(buffer: *mut c_void, _ctx: *mut c_void) {
+unsafe extern "C" fn header_lowertransform(_det: *mut c_void, buffer: *mut c_void, _ctx: *mut c_void) {
     let input = InspectionBufferPtr(buffer);
     let input_len = InspectionBufferLength(buffer);
     if input.is_null() || input_len == 0 {
@@ -114,7 +114,7 @@ fn strip_pseudo_transform_do(input: &[u8], output: &mut [u8]) -> u32 {
     return nb as u32;
 }
 
-unsafe extern "C" fn strip_pseudo_transform(buffer: *mut c_void, _ctx: *mut c_void) {
+unsafe extern "C" fn strip_pseudo_transform(_det: *mut c_void, buffer: *mut c_void, _ctx: *mut c_void) {
     let input = InspectionBufferPtr(buffer);
     let input_len = InspectionBufferLength(buffer);
     if input.is_null() || input_len == 0 {

rust/src/detect/transforms/mod.rs

@@ -37,7 +37,7 @@ pub struct SCTransformTableElmt {
     pub flags: u16,
     pub Setup: unsafe extern "C" fn(de: *mut c_void, s: *mut c_void, raw: *const c_char) -> c_int,
     pub Free: Option<unsafe extern "C" fn(de: *mut c_void, ptr: *mut c_void)>,
-    pub Transform: unsafe extern "C" fn(inspect_buf: *mut c_void, options: *mut c_void),
+    pub Transform: unsafe extern "C" fn(det: *mut c_void, inspect_buf: *mut c_void, options: *mut c_void),
     pub TransformValidate:
         Option<unsafe extern "C" fn(content: *const u8, len: u16, context: *mut c_void) -> bool>,
 }

rust/src/detect/transforms/strip_whitespace.rs

@@ -46,7 +46,7 @@ fn strip_whitespace_transform_do(input: &[u8], output: &mut [u8]) -> u32 {
     return nb as u32;
 }
 
-unsafe extern "C" fn strip_whitespace_transform(buffer: *mut c_void, _ctx: *mut c_void) {
+unsafe extern "C" fn strip_whitespace_transform(_det: *mut c_void, buffer: *mut c_void, _ctx: *mut c_void) {
     let input = InspectionBufferPtr(buffer);
     let input_len = InspectionBufferLength(buffer);
     if input.is_null() || input_len == 0 {

rust/src/detect/transforms/urldecode.rs

@@ -86,7 +86,7 @@ fn url_decode_transform_do(input: &[u8], output: &mut [u8]) -> u32 {
     return nb as u32;
 }
 
-unsafe extern "C" fn url_decode_transform(buffer: *mut c_void, _ctx: *mut c_void) {
+unsafe extern "C" fn url_decode_transform(_det: *mut c_void, buffer: *mut c_void, _ctx: *mut c_void) {
     let input = InspectionBufferPtr(buffer);
     let input_len = InspectionBufferLength(buffer);
     if input.is_null() || input_len == 0 {

rust/src/detect/transforms/xor.rs

@@ -80,7 +80,7 @@ fn xor_transform_do(input: &[u8], output: &mut [u8], ctx: &DetectTransformXorDat
     }
 }
 
-unsafe extern "C" fn xor_transform(buffer: *mut c_void, ctx: *mut c_void) {
+unsafe extern "C" fn xor_transform(_det: *mut c_void, buffer: *mut c_void, ctx: *mut c_void) {
     let input = InspectionBufferPtr(buffer);
     let input_len = InspectionBufferLength(buffer);
     if input.is_null() || input_len == 0 {

src/detect-dce-stub-data.c

@@ -80,7 +80,7 @@ static InspectionBuffer *GetSMBData(DetectEngineThreadCtx *det_ctx,
         SCLogDebug("have data!");
 
         InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
-        InspectionBufferApplyTransforms(buffer, transforms);
+        InspectionBufferApplyTransforms(det_ctx, buffer, transforms);
     }
     return buffer;
 }
@@ -106,7 +106,7 @@ static InspectionBuffer *GetDCEData(DetectEngineThreadCtx *det_ctx,
             buffer->flags |= DETECT_CI_FLAGS_DCE_BE;
         }
         InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
-        InspectionBufferApplyTransforms(buffer, transforms);
+        InspectionBufferApplyTransforms(det_ctx, buffer, transforms);
     }
     return buffer;
 }

src/detect-dnp3.c

@@ -167,7 +167,7 @@ static InspectionBuffer *GetDNP3Data(DetectEngineThreadCtx *det_ctx,
 
         SCLogDebug("tx %p data %p data_len %u", tx, tx->buffer, tx->buffer_len);
         InspectionBufferSetup(det_ctx, list_id, buffer, tx->buffer, tx->buffer_len);
-        InspectionBufferApplyTransforms(buffer, transforms);
+        InspectionBufferApplyTransforms(det_ctx, buffer, transforms);
     }
     return buffer;
 }

src/detect-dns-answer-name.c

@@ -64,7 +64,7 @@ static InspectionBuffer *GetBuffer(DetectEngineThreadCtx *det_ctx,
         InspectionBufferSetupMultiEmpty(buffer);
         return NULL;
     }
-    InspectionBufferSetupMulti(buffer, transforms, data, data_len);
+    InspectionBufferSetupMulti(det_ctx, buffer, transforms, data, data_len);
     buffer->flags = DETECT_CI_FLAGS_SINGLE;
     return buffer;
 }

src/detect-dns-query-name.c

@@ -64,7 +64,7 @@ static InspectionBuffer *GetBuffer(DetectEngineThreadCtx *det_ctx,
         InspectionBufferSetupMultiEmpty(buffer);
         return NULL;
     }
-    InspectionBufferSetupMulti(buffer, transforms, data, data_len);
+    InspectionBufferSetupMulti(det_ctx, buffer, transforms, data, data_len);
     buffer->flags = DETECT_CI_FLAGS_SINGLE;
     return buffer;
 }

src/detect-dns-query.c

@@ -85,7 +85,7 @@ static InspectionBuffer *DnsQueryGetData(DetectEngineThreadCtx *det_ctx,
         InspectionBufferSetupMultiEmpty(buffer);
         return NULL;
     }
-    InspectionBufferSetupMulti(buffer, transforms, data, data_len);
+    InspectionBufferSetupMulti(det_ctx, buffer, transforms, data, data_len);
     buffer->flags = DETECT_CI_FLAGS_SINGLE;
 
     SCReturnPtr(buffer, "InspectionBuffer");

src/detect-engine-frame.c

@@ -67,8 +67,8 @@ static bool SetupStreamCallbackData(struct FrameStreamData *dst, const TcpSessio
 
 static bool BufferSetup(struct FrameStreamData *fsd, InspectionBuffer *buffer, const uint8_t *input,
         const uint32_t input_len, const uint64_t input_offset);
-static void BufferSetupUdp(InspectionBuffer *buffer, const Frame *frame, const Packet *p,
-        const DetectEngineTransforms *transforms);
+static void BufferSetupUdp(DetectEngineThreadCtx *det_ctx, InspectionBuffer *buffer,
+        const Frame *frame, const Packet *p, const DetectEngineTransforms *transforms);
 
 void DetectRunPrefilterFrame(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, Packet *p,
         const Frames *frames, const Frame *frame, const AppProto alproto)
@@ -159,7 +159,7 @@ static void PrefilterMpmFrame(DetectEngineThreadCtx *det_ctx, const void *pectx,
         if (frame->offset >= p->payload_len)
             return;
 
-        BufferSetupUdp(buffer, frame, p, ctx->transforms);
+        BufferSetupUdp(det_ctx, buffer, frame, p, ctx->transforms);
         const uint32_t data_len = buffer->inspect_len;
         const uint8_t *data = buffer->inspect;
 
@@ -251,8 +251,8 @@ bool DetectRunFrameInspectRule(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, c
     return false;
 }
 
-static void BufferSetupUdp(InspectionBuffer *buffer, const Frame *frame, const Packet *p,
-        const DetectEngineTransforms *transforms)
+static void BufferSetupUdp(DetectEngineThreadCtx *det_ctx, InspectionBuffer *buffer,
+        const Frame *frame, const Packet *p, const DetectEngineTransforms *transforms)
 {
     uint8_t ci_flags = DETECT_CI_FLAGS_START;
     uint32_t frame_len;
@@ -275,7 +275,7 @@ static void BufferSetupUdp(InspectionBuffer *buffer, const Frame *frame, const P
             AppLayerParserGetFrameNameById(p->flow->proto, p->flow->alproto, frame->type),
             frame->offset, frame->type, frame->len);
 
-    InspectionBufferSetupMulti(buffer, transforms, data, data_len);
+    InspectionBufferSetupMulti(det_ctx, buffer, transforms, data, data_len);
     buffer->inspect_offset = 0;
     buffer->flags = ci_flags;
 }
@@ -301,7 +301,7 @@ static int DetectFrameInspectUdp(DetectEngineThreadCtx *det_ctx,
         return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
 
     if (!buffer->initialized)
-        BufferSetupUdp(buffer, frame, p, transforms);
+        BufferSetupUdp(det_ctx, buffer, frame, p, transforms);
     DEBUG_VALIDATE_BUG_ON(!buffer->initialized);
     if (buffer->inspect == NULL)
         return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
@@ -387,7 +387,7 @@ static bool BufferSetup(struct FrameStreamData *fsd, InspectionBuffer *buffer, c
     }
     // PrintRawDataFp(stdout, data, data_len);
     SCLogDebug("fsd->transforms %p", fsd->transforms);
-    InspectionBufferSetupMulti(buffer, fsd->transforms, data, data_len);
+    InspectionBufferSetupMulti(fsd->det_ctx, buffer, fsd->transforms, data, data_len);
     SCLogDebug("inspect_offset %" PRIu64, fo_inspect_offset);
     buffer->inspect_offset = fo_inspect_offset;
     buffer->flags = ci_flags;

src/detect-engine-helper.c

@@ -57,7 +57,7 @@ InspectionBuffer *DetectHelperGetData(struct DetectEngineThreadCtx_ *det_ctx,
             return NULL;
 
         InspectionBufferSetup(det_ctx, list_id, buffer, b, b_len);
-        InspectionBufferApplyTransforms(buffer, transforms);
+        InspectionBufferApplyTransforms(det_ctx, buffer, transforms);
     }
     return buffer;
 }
@@ -137,8 +137,9 @@ int DetectHelperTransformRegister(const SCTransformTableElmt *kw)
     sigmatch_table[DETECT_TBLSIZE_IDX].desc = kw->desc;
     sigmatch_table[DETECT_TBLSIZE_IDX].url = kw->url;
     sigmatch_table[DETECT_TBLSIZE_IDX].flags = kw->flags;
-    sigmatch_table[DETECT_TBLSIZE_IDX].Transform =
-            (void (*)(InspectionBuffer * buffer, void *options)) kw->Transform;
+    sigmatch_table[DETECT_TBLSIZE_IDX].Transform = (void (*)(
+            struct DetectEngineThreadCtx_ * det_ctx, InspectionBuffer * buffer, void *options))
+                                                           kw->Transform;
     sigmatch_table[DETECT_TBLSIZE_IDX].TransformValidate = (bool (*)(
             const uint8_t *content, uint16_t content_len, void *context))kw->TransformValidate;
     sigmatch_table[DETECT_TBLSIZE_IDX].Setup =
@@ -167,7 +168,7 @@ InspectionBuffer *DetectHelperGetMultiData(struct DetectEngineThreadCtx_ *det_ct
         InspectionBufferSetupMultiEmpty(buffer);
         return NULL;
     }
-    InspectionBufferSetupMulti(buffer, transforms, data, data_len);
+    InspectionBufferSetupMulti(det_ctx, buffer, transforms, data, data_len);
     buffer->flags = DETECT_CI_FLAGS_SINGLE;
     return buffer;
 }

src/detect-engine-mpm.h

@@ -113,7 +113,6 @@ void DetectEngineFrameMpmRegister(DetectEngineCtx *de_ctx, const char *name, int
                 const DetectBufferMpmRegistry *mpm_reg, int list_id),
         AppProto alproto, uint8_t type);
 
-
 int PrefilterGenericMpmFrameRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh, MpmCtx *mpm_ctx,
         const DetectBufferMpmRegistry *mpm_reg, int list_id);
 

src/detect-engine.c

@@ -960,14 +960,15 @@ static char DetectBufferTypeCompareIdFunc(void *data1, uint16_t len1, void *data
     return map1->id == map2->id;
 }
 
-static void DetectBufferTypeFreeFunc(void *data)
+static void DetectBufferTypeFreeFuncWithCtx(void *ctx, void *data)
 {
-    DetectBufferType *map = (DetectBufferType *)data;
-
-    if (map == NULL) {
+    if (data == NULL) {
         return;
     }
 
+    DetectBufferType *map = (DetectBufferType *)data;
+    DetectEngineCtx *de_ctx = (DetectEngineCtx *)ctx;
+
     /* Release transformation option memory, if any */
     for (int i = 0; i < map->transforms.cnt; i++) {
         if (map->transforms.transforms[i].options == NULL)
@@ -977,12 +978,18 @@ static void DetectBufferTypeFreeFunc(void *data)
                     sigmatch_table[map->transforms.transforms[i].transform].name);
             continue;
         }
-        sigmatch_table[map->transforms.transforms[i].transform].Free(NULL, map->transforms.transforms[i].options);
+        sigmatch_table[map->transforms.transforms[i].transform].Free(
+                de_ctx, map->transforms.transforms[i].options);
     }
 
     SCFree(map);
 }
 
+static void DetectBufferTypeFreeFunc(void *data)
+{
+    DetectBufferTypeFreeFuncWithCtx(NULL, data);
+}
+
 static int DetectBufferTypeInit(void)
 {
     BUG_ON(g_buffer_type_hash);
@@ -1580,8 +1587,8 @@ void InspectionBufferSetupMultiEmpty(InspectionBuffer *buffer)
 }
 
 /** \brief setup the buffer with our initial data */
-void InspectionBufferSetupMulti(InspectionBuffer *buffer, const DetectEngineTransforms *transforms,
-        const uint8_t *data, const uint32_t data_len)
+void InspectionBufferSetupMulti(DetectEngineThreadCtx *det_ctx, InspectionBuffer *buffer,
+        const DetectEngineTransforms *transforms, const uint8_t *data, const uint32_t data_len)
 {
 #ifdef DEBUG_VALIDATION
     DEBUG_VALIDATE_BUG_ON(!buffer->multi);
@@ -1591,7 +1598,7 @@ void InspectionBufferSetupMulti(InspectionBuffer *buffer, const DetectEngineTran
     buffer->len = 0;
     buffer->initialized = true;
 
-    InspectionBufferApplyTransforms(buffer, transforms);
+    InspectionBufferApplyTransforms(det_ctx, buffer, transforms);
 }
 
 /** \brief setup the buffer with our initial data */
@@ -1711,7 +1718,7 @@ bool DetectEngineBufferTypeValidateTransform(DetectEngineCtx *de_ctx, int sm_lis
     return true;
 }
 
-void InspectionBufferApplyTransforms(InspectionBuffer *buffer,
+void InspectionBufferApplyTransforms(DetectEngineThreadCtx *det_ctx, InspectionBuffer *buffer,
         const DetectEngineTransforms *transforms)
 {
     if (transforms) {
@@ -1720,7 +1727,7 @@ void InspectionBufferApplyTransforms(InspectionBuffer *buffer,
             if (id == 0)
                 break;
             BUG_ON(sigmatch_table[id].Transform == NULL);
-            sigmatch_table[id].Transform(buffer, transforms->transforms[i].options);
+            sigmatch_table[id].Transform(det_ctx, buffer, transforms->transforms[i].options);
             SCLogDebug("applied transform %s", sigmatch_table[id].name);
         }
     }
@@ -1731,8 +1738,8 @@ static void DetectBufferTypeSetupDetectEngine(DetectEngineCtx *de_ctx)
     const int size = g_buffer_type_id;
     BUG_ON(!(size > 0));
 
-    de_ctx->buffer_type_hash_name = HashListTableInit(256, DetectBufferTypeHashNameFunc,
-            DetectBufferTypeCompareNameFunc, DetectBufferTypeFreeFunc);
+    de_ctx->buffer_type_hash_name = HashListTableInitWithCtx(256, DetectBufferTypeHashNameFunc,
+            DetectBufferTypeCompareNameFunc, DetectBufferTypeFreeFuncWithCtx);
     BUG_ON(de_ctx->buffer_type_hash_name == NULL);
     de_ctx->buffer_type_hash_id =
             HashListTableInit(256, DetectBufferTypeHashIdFunc, DetectBufferTypeCompareIdFunc,
@@ -1773,7 +1780,7 @@ static void DetectBufferTypeFreeDetectEngine(DetectEngineCtx *de_ctx)
 {
     if (de_ctx) {
         if (de_ctx->buffer_type_hash_name)
-            HashListTableFree(de_ctx->buffer_type_hash_name);
+            HashListTableFreeWithCtx(de_ctx, de_ctx->buffer_type_hash_name);
         if (de_ctx->buffer_type_hash_id)
             HashListTableFree(de_ctx->buffer_type_hash_id);
 
@@ -2599,6 +2606,9 @@ DetectEngineCtx *DetectEngineCtxInitWithPrefix(const char *prefix, uint32_t tena
 static void DetectEngineCtxFreeThreadKeywordData(DetectEngineCtx *de_ctx)
 {
     HashListTableFree(de_ctx->keyword_hash);
+#if UNITTESTS
+    de_ctx->keyword_hash = NULL;
+#endif
 }
 
 static void DetectEngineCtxFreeFailedSigs(DetectEngineCtx *de_ctx)
@@ -2671,7 +2681,6 @@ void DetectEngineCtxFree(DetectEngineCtx *de_ctx)
 
     MpmFactoryDeRegisterAllMpmCtxProfiles(de_ctx);
 
-    DetectEngineCtxFreeThreadKeywordData(de_ctx);
     SRepDestroy(de_ctx);
     DetectEngineCtxFreeFailedSigs(de_ctx);
 
@@ -2694,6 +2703,7 @@ void DetectEngineCtxFree(DetectEngineCtx *de_ctx)
     DetectPortCleanupList(de_ctx, de_ctx->udp_priorityports);
 
     DetectBufferTypeFreeDetectEngine(de_ctx);
+    DetectEngineCtxFreeThreadKeywordData(de_ctx);
     SCClassConfDeinit(de_ctx);
     SCReferenceConfDeinit(de_ctx);