Next/20200606/v6

doc/userguide/rules/header-keywords.rst

@@ -259,6 +259,13 @@ field. The tos keyword can be have a value from 0 - 255. This field of the
 IP header has been updated by `rfc2474 <https://tools.ietf.org/html/rfc2474>`_
 to include functionality for
 `Differentiated services <https://en.wikipedia.org/wiki/Differentiated_services>`_.
+Note that the value of the field has been defined with the right-most 2 bits having
+the value 0. When specifying a value for tos, ensure that the value follows this.
+
+E.g, instead of specifying the decimal value 34 (hex 22), right shift twice and use
+decimal 136 (hex 88).
+
+You can specify hexadecimal values as with a leading `x`, e.g, `x88`.
 
 Format of tos::
 

doc/userguide/rules/transforms.rst

@@ -1,7 +1,8 @@
 Transformations
 ===============
 
-Transformation keywords turn the data at a sticky buffer into something else.
+Transformation keywords turn the data at a sticky buffer into something else. Some transformations
+support options for greater control over the transformation process
 
 Example::
 
@@ -12,7 +13,7 @@ This example will match on traffic even if there are one or more spaces between
 the ``navigate`` and ``(``.
 
 The transforms can be chained. They are processed in the order in which they
-appear in a rule. Each transforms output acts as input for the next one.
+appear in a rule. Each transform's output acts as input for the next one.
 
 Example::
 
@@ -106,3 +107,16 @@ Example::
 
 .. note:: depends on libnss being compiled into Suricata
 
+pcrexform
+---------
+
+Takes the buffer, applies the required regular expression, and outputs the *first captured expression*.
+
+.. note:: this transform requires a mandatory option string containing a regular expression.
+
+
+This example alerts if ``http.request_line`` contains ``/dropper.php``
+Example::
+
+    alert http any any -> any any (msg:"HTTP with pcrexform"; http.request_line; \
+        pcrexform:"[a-zA-Z]+\s+(.*)\s+HTTP"; content:"/dropper.php"; sid:1;)

src/Makefile.am

@@ -291,6 +291,7 @@ detect-transform-md5.c detect-transform-md5.h \
 detect-transform-sha1.c detect-transform-sha1.h \
 detect-transform-sha256.c detect-transform-sha256.h \
 detect-transform-dotprefix.c detect-transform-dotprefix.h \
+detect-transform-pcrexform.c detect-transform-pcrexform.h \
 detect-ttl.c detect-ttl.h \
 detect-uricontent.c detect-uricontent.h \
 detect-urilen.c detect-urilen.h \

src/app-layer-htp.c

@@ -1928,9 +1928,9 @@ static int HTPCallbackRequestBodyData(htp_tx_data_t *d)
             HtpRequestBodyHandleMultipart(hstate, tx_ud, d->tx, chunks_buffer, chunks_buffer_len);
 
         } else if (tx_ud->request_body_type == HTP_BODY_REQUEST_POST) {
-            HtpRequestBodyHandlePOST(hstate, tx_ud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
+            HtpRequestBodyHandlePOST(hstate, tx_ud, d->tx, (uint8_t *)d->data, len);
         } else if (tx_ud->request_body_type == HTP_BODY_REQUEST_PUT) {
-            HtpRequestBodyHandlePUT(hstate, tx_ud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
+            HtpRequestBodyHandlePUT(hstate, tx_ud, d->tx, (uint8_t *)d->data, len);
         }
 
     } else {
@@ -2027,7 +2027,7 @@ static int HTPCallbackResponseBodyData(htp_tx_data_t *d)
 
         HtpBodyAppendChunk(&hstate->cfg->response, &tx_ud->response_body, d->data, len);
 
-        HtpResponseBodyHandle(hstate, tx_ud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
+        HtpResponseBodyHandle(hstate, tx_ud, d->tx, (uint8_t *)d->data, len);
     } else {
         if (tx_ud->tcflags & HTP_FILENAME_SET) {
             SCLogDebug("closing file that was being stored");

src/detect-asn1.c

@@ -148,15 +148,28 @@ static int DetectAsn1Match(DetectEngineThreadCtx *det_ctx, Packet *p,
     }
 
     const DetectAsn1Data *ad = (const DetectAsn1Data *)ctx;
-    int32_t offset;
+    uint16_t offset;
     if (ad->flags & ASN1_ABSOLUTE_OFFSET) {
         offset = ad->absolute_offset;
     } else if (ad->flags & ASN1_RELATIVE_OFFSET) {
-        offset = ad->relative_offset;
+        // relative offset in regards to the last content match
+
+        // This range check is done when relative_offset
+        BUG_ON(ad->relative_offset > UINT16_MAX || ad->relative_offset < -UINT16_MAX);
+
+        int64_t tmp_offset = det_ctx->buffer_offset + ad->relative_offset;
+
+        // check for under/overflow before downcasting
+        if (tmp_offset < 0 || tmp_offset > UINT16_MAX) {
+            return 0;
+        }
+
+        offset = (uint16_t)tmp_offset;
     } else {
         offset = 0;
     }
-    if (offset >= (int32_t)p->payload_len) {
+
+    if (offset >= p->payload_len) {
         return 0;
     }
 
@@ -204,7 +217,7 @@ static DetectAsn1Data *DetectAsn1Parse(const char *instr)
     DetectAsn1Data *fd = NULL;
     char *tok = NULL;
     uint32_t ov_len = 0;
-    uint32_t abs_off = 0;
+    uint16_t abs_off = 0;
     int32_t rel_off = 0;
     uint8_t flags = 0;
     char *saveptr = NULL;
@@ -244,7 +257,7 @@ static DetectAsn1Data *DetectAsn1Parse(const char *instr)
             /* get the param */
             tok = strtok_r(NULL, ASN_DELIM, &saveptr);
             if (tok == NULL ||
-                StringParseUint32(&abs_off, 10, 0, tok) <= 0)
+                StringParseUint16(&abs_off, 10, 0, tok) <= 0)
             {
                 SCLogError(SC_ERR_INVALID_VALUE, "Malformed value for "
                            "absolute_offset: %s", tok);
@@ -254,8 +267,11 @@ static DetectAsn1Data *DetectAsn1Parse(const char *instr)
             flags |= ASN1_RELATIVE_OFFSET;
             /* get the param */
             tok = strtok_r(NULL, ASN_DELIM, &saveptr);
+
+            // Range check on uint16_t max as uint16_t is the type of the buffer
+            // the offset is relative to
             if (tok == NULL ||
-                StringParseInt32(&rel_off, 10, 0, tok) <= 0)
+                StringParseI32RangeCheck(&rel_off, 10, 0, tok, -UINT16_MAX, UINT16_MAX) <= 0)
             {
                 SCLogError(SC_ERR_INVALID_VALUE, "Malformed value for "
                            "relative_offset: %s", tok);
@@ -1092,7 +1108,7 @@ static int DetectAsn1TestReal01(void)
              "content:\"Pablo\"; asn1:absolute_offset 0, "
              "oversize_length 130; sid:1;)";
     sigs[1]= "alert ip any any -> any any (msg:\"Testing id 2\"; "
-             "content:\"AA\"; asn1:relative_offset 2, "
+             "content:\"AA\"; asn1:relative_offset 0, "
              "oversize_length 130; sid:2;)";
     sigs[2]= "alert ip any any -> any any (msg:\"Testing id 3\"; "
              "content:\"lalala\"; asn1: oversize_length 2000; sid:3;)";
@@ -1171,7 +1187,7 @@ static int DetectAsn1TestReal02(void)
              "content:\"Pablo\"; asn1:absolute_offset 0, "
              "oversize_length 140; sid:1;)";
     sigs[1]= "alert ip any any -> any any (msg:\"Testing id 2\"; "
-             "content:\"AA\"; asn1:relative_offset 2, "
+             "content:\"AA\"; asn1:relative_offset 0, "
              "oversize_length 140; sid:2;)";
     sigs[2]= "alert ip any any -> any any (msg:\"Testing id 3\"; "
              "content:\"lalala\"; asn1: oversize_length 2000; sid:3;)";
@@ -1252,7 +1268,7 @@ static int DetectAsn1TestReal03(void)
 /**
  * \test DetectAsn1TestReal04 like the real test 02, but modified the
  *       relative offset to check negative offset values, in this case
- *       start decoding from -7 bytes respect the content match "John"
+ *       start decoding from -11 bytes respect the content match "John"
  */
 static int DetectAsn1TestReal04(void)
 {
@@ -1309,7 +1325,7 @@ static int DetectAsn1TestReal04(void)
              "content:\"Pablo\"; asn1:absolute_offset 0, "
              "oversize_length 140; sid:1;)";
     sigs[1]= "alert ip any any -> any any (msg:\"Testing id 2\"; "
-             "content:\"John\"; asn1:relative_offset -7, "
+             "content:\"John\"; asn1:relative_offset -11, "
              "oversize_length 140; sid:2;)";
     sigs[2]= "alert ip any any -> any any (msg:\"Testing id 3\"; "
              "content:\"lalala\"; asn1: oversize_length 2000; sid:3;)";
@@ -1328,6 +1344,64 @@ static int DetectAsn1TestReal04(void)
     return result;
 }
 
+/**
+ * \test DetectAsn1TestReal05 like the real test 04, but modified the
+ *       relative offset to check offset values which could read past the bounds
+ *       of the buffer
+ */
+static int DetectAsn1TestReal05(void)
+{
+    int result = 0;
+    /* Check the start with AA (this is to test the relative_offset keyword) */
+    uint8_t *buf = (uint8_t *) "AA\x60\x81\x85\x61\x10\x1A\x04""John""\x1A\x01"
+                   "P""\x1A\x05""Smith""\xA0\x0A\x1A\x08""Director"
+                   "\x42\x01\x33\xA1\x0A\x43\x08""19710917"
+                   "\xA2\x12\x61\x10\x1A\x04""Mary""\x1A\x01""T""\x1A\x05"
+                   "Smith""\xA3\x42\x31\x1F\x61\x11\x1A\x05""Ralph""\x1A\x01"
+                   "T""\x1A\x05""Smith""\xA0\x0A\x43\x08""19571111"
+                   "\x31\x1F\x61\x11\x1A\x05""Susan""\x1A\x01""B""\x1A\x05"
+                   "Jones""\xA0\x0A\x43\x08""19590717"
+                   "\x60\x81\x85\x61\x10\x1A\x04""John""\x1A\x01""P"
+                   "\x1A\x05""Smith""\xA0\x0A\x1A\x08""Director"
+                   "\x42\x01\x33\xA1\x0A\x43\x08""19710917"
+                   "\xA2\x12\x61\x10\x1A\x04""Mary""\x1A\x01""T""\x1A\x05"
+                   "Smith""\xA3\x42\x31\x1F\x61\x11\x1A\x05""Ralph""\x1A\x01"
+                   "T""\x1A\x05""Smith""\xA0\x0A\x43\x08""19571111""\x31\x1F"
+                   "\x61\x11\x1A\x05""Susan""\x1A\x01""B""\x1A\x05""Jones"
+                   "\xA0\x0A\x43\x08""19590717";
+
+    uint16_t buflen = strlen((char *)buf) - 1;
+
+    Packet *p[1];
+
+    p[0] = UTHBuildPacket((uint8_t *)buf, buflen, IPPROTO_TCP);
+
+    if (p[0] == NULL)
+        goto end;
+
+    const char *sigs[3];
+    sigs[0]= "alert ip any any -> any any (msg:\"Testing id 1\"; "
+             "content:\"John\"; asn1:relative_offset -100, "
+             "oversize_length 132; sid:1;)";
+    sigs[1]= "alert ip any any -> any any (msg:\"Testing id 2\"; "
+             "content:\"John\"; asn1:relative_offset -11, "
+             "oversize_length 132; sid:2;)";
+    sigs[2]= "alert ip any any -> any any (msg:\"Testing id 3\"; "
+             "content:\"John\"; asn1:  relative_offset 5000, "
+             "oversize_length 132; sid:3;)";
+
+    uint32_t sid[3] = {1, 2, 3};
+
+    // The second signature should match
+    uint32_t results[1][3] = {{0, 1, 0}};
+
+    result = UTHGenericTest(p, 1, sigs, sid, (uint32_t *) results, 3);
+
+    UTHFreePackets(p, 1);
+end:
+    return result;
+}
+
 #endif /* UNITTESTS */
 
 /**
@@ -1364,6 +1438,7 @@ static void DetectAsn1RegisterTests(void)
     UtRegisterTest("DetectAsn1TestReal02", DetectAsn1TestReal02);
     UtRegisterTest("DetectAsn1TestReal03", DetectAsn1TestReal03);
     UtRegisterTest("DetectAsn1TestReal04", DetectAsn1TestReal04);
+    UtRegisterTest("DetectAsn1TestReal05", DetectAsn1TestReal05);
 
 #endif /* UNITTESTS */
 }

src/detect-asn1.h

@@ -36,12 +36,11 @@
 typedef struct DetectAsn1Data_ {
     uint8_t flags;     /* flags indicating the checks loaded */
     uint32_t oversize_length;   /* Length argument if needed */
-    int32_t absolute_offset;   /* Length argument if needed */
+    uint16_t absolute_offset;   /* Length argument if needed */
     int32_t relative_offset;   /* Length argument if needed */
 } DetectAsn1Data;
 
 /* prototypes */
 void DetectAsn1Register (void);
 
 #endif /* __DETECT_ASN1_H__ */
-

src/detect-engine-mpm.c

@@ -173,7 +173,7 @@ void DetectAppLayerMpmRegisterByParentId(DetectEngineCtx *de_ctx,
                 for (int i = 0; i < transforms->cnt; i++) {
                     char ttstr[64];
                     (void)snprintf(ttstr,sizeof(ttstr), "%s,",
-                            sigmatch_table[transforms->transforms[i]].name);
+                            sigmatch_table[transforms->transforms[i].transform].name);
                     strlcat(xforms, ttstr, sizeof(xforms));
                 }
                 xforms[strlen(xforms)-1] = '\0';

src/detect-engine-register.c

@@ -207,6 +207,7 @@
 #include "detect-transform-sha1.h"
 #include "detect-transform-sha256.h"
 #include "detect-transform-dotprefix.h"
+#include "detect-transform-pcrexform.h"
 
 #include "util-rule-vars.h"
 
@@ -582,6 +583,7 @@ void SigTableSetup(void)
     DetectTransformSha1Register();
     DetectTransformSha256Register();
     DetectTransformDotPrefixRegister();
+    DetectTransformPcrexformRegister();
 
     /* close keyword registration */
     DetectBufferTypeCloseRegistration();

src/detect-engine-register.h

@@ -263,6 +263,7 @@ enum DetectKeywordId {
     DETECT_TRANSFORM_SHA1,
     DETECT_TRANSFORM_SHA256,
     DETECT_TRANSFORM_DOTPREFIX,
+    DETECT_TRANSFORM_PCREXFORM,
 
     /* make sure this stays last */
     DETECT_TBLSIZE,

src/detect-engine.c

@@ -730,7 +730,10 @@ static HashListTable *g_buffer_type_hash = NULL;
 static int g_buffer_type_id = DETECT_SM_LIST_DYNAMIC_START;
 static int g_buffer_type_reg_closed = 0;
 
-static DetectEngineTransforms no_transforms = { .transforms = { 0 }, .cnt = 0, };
+static DetectEngineTransforms no_transforms = {
+    .transforms[0] = {0, NULL},
+    .cnt = 0,
+};
 
 int DetectBufferTypeMaxId(void)
 {
@@ -763,6 +766,19 @@ static char DetectBufferTypeCompareFunc(void *data1, uint16_t len1, void *data2,
 static void DetectBufferTypeFreeFunc(void *data)
 {
     DetectBufferType *map = (DetectBufferType *)data;
+
+    /* Release transformation option memory, if any */
+    for (int i = 0; i < map->transforms.cnt; i++) {
+        if (map->transforms.transforms[i].options == NULL)
+            continue;
+        if (sigmatch_table[map->transforms.transforms[i].transform].Free == NULL) {
+            SCLogError(SC_ERR_UNIMPLEMENTED,
+                       "%s allocates transform option memory but has no free routine",
+                       sigmatch_table[map->transforms.transforms[i].transform].name);
+            continue;
+        }
+        sigmatch_table[map->transforms.transforms[i].transform].Free(NULL, map->transforms.transforms[i].options);
+    }
     if (map != NULL) {
         SCFree(map);
     }
@@ -973,7 +989,7 @@ int DetectBufferSetActiveList(Signature *s, const int list)
 {
     BUG_ON(s->init_data == NULL);
 
-    if (s->init_data->list && s->init_data->transform_cnt) {
+    if (s->init_data->list && s->init_data->transforms.cnt) {
         return -1;
     }
     s->init_data->list = list;
@@ -986,27 +1002,27 @@ int DetectBufferGetActiveList(DetectEngineCtx *de_ctx, Signature *s)
 {
     BUG_ON(s->init_data == NULL);
 
-    if (s->init_data->transform_cnt) {
+    if (s->init_data->list && s->init_data->transforms.cnt) {
         if (s->init_data->list == DETECT_SM_LIST_NOTSET ||
             s->init_data->list < DETECT_SM_LIST_DYNAMIC_START) {
             SCLogError(SC_ERR_INVALID_SIGNATURE, "previous transforms not consumed "
                     "(list: %u, transform_cnt %u)", s->init_data->list,
-                    s->init_data->transform_cnt);
+                    s->init_data->transforms.cnt);
             SCReturnInt(-1);
         }
 
         SCLogDebug("buffer %d has transform(s) registered: %d",
-                s->init_data->list, s->init_data->transforms[0]);
+                s->init_data->list, s->init_data->transforms.cnt);
         int new_list = DetectBufferTypeGetByIdTransforms(de_ctx, s->init_data->list,
-                s->init_data->transforms, s->init_data->transform_cnt);
+                s->init_data->transforms.transforms, s->init_data->transforms.cnt);
         if (new_list == -1) {
             SCReturnInt(-1);
         }
         SCLogDebug("new_list %d", new_list);
         s->init_data->list = new_list;
         s->init_data->list_set = false;
         // reset transforms now that we've set up the list
-        s->init_data->transform_cnt = 0;
+        s->init_data->transforms.cnt = 0;
     }
 
     SCReturnInt(0);
@@ -1150,11 +1166,11 @@ void InspectionBufferApplyTransforms(InspectionBuffer *buffer,
 {
     if (transforms) {
         for (int i = 0; i < DETECT_TRANSFORMS_MAX; i++) {
-            const int id = transforms->transforms[i];
+            const int id = transforms->transforms[i].transform;
             if (id == 0)
                 break;
             BUG_ON(sigmatch_table[id].Transform == NULL);
-            sigmatch_table[id].Transform(buffer);
+            sigmatch_table[id].Transform(buffer, transforms->transforms[i].options);
             SCLogDebug("applied transform %s", sigmatch_table[id].name);
         }
     }
@@ -1242,7 +1258,7 @@ void DetectBufferTypeCloseRegistration(void)
 }
 
 int DetectBufferTypeGetByIdTransforms(DetectEngineCtx *de_ctx, const int id,
-        int *transforms, int transform_cnt)
+        TransformData *transforms, int transform_cnt)
 {
     const DetectBufferType *base_map = DetectBufferTypeGetById(de_ctx, id);
     if (!base_map) {