original packet include options, decoding file hash fields

Author: skhademcis

estreamer/adapters/splunk.py

@@ -111,7 +111,7 @@
         View.NETWORK_ANALYSIS_POLICY: 'net_analysis_policy',
         View.ORIGINAL_CLIENT_SRC_IP: 'original_src_ip',
         View.PACKET_DATA: 'packet',
-        View.PACKET_DATA_FULL: 'packetHex',
+        View.PACKET_DATA_FULL: 'original_packet',
         View.PARENT_DETECTION: 'parent_detection',
         View.PRIORITY: 'priority',
         View.PROTOCOL: 'protocol',

estreamer/metadata/cache.py

@@ -83,7 +83,7 @@ class Cache( object ):
     ORIGINAL_SRC_IP = 'originalSrcIp'
     OS_FINGERPRINTS = 'osFingerprints'
     PACKET_DATA = 'packet'
-    PACKET_DATA_FULL = 'packetHex'
+    PACKET_DATA_FULL = 'originalPacket'
     PAYLOADS = 'payloads'
     POLICIES = 'policies'
     PRIORITIES = 'priorities'
@@ -151,7 +151,7 @@ class Cache( object ):
         NET_PROTOS: 'net_protos',
         OS_FINGERPRINTS: 'os_fingerprints',
         PACKET_DATA: 'packet',
-        PACKET_DATA_FULL: 'packetHex',
+        PACKET_DATA_FULL: 'originalPacket',
         PAYLOADS: 'payloads',
         POLICIES: 'policies',
         PRIORITIES: 'priorities',

estreamer/metadata/view.py

@@ -110,7 +110,7 @@ class View( object ):
     NETWORK_ANALYSIS_POLICY = 'networkAnalysisPolicy'
     ORIGINAL_CLIENT_SRC_IP = 'originalSrcIP'
     PACKET_DATA = 'packet'
-    PACKET_DATA_FULL = 'packetHex'
+    PACKET_DATA_FULL = 'originalPacket'
     PARENT_DETECTION = 'parentDetection'
     PRIORITY = 'priority'
     PROTOCOL = 'protocol'
@@ -345,7 +345,6 @@ def create( self ):
 
             packet = record['packetData']
             packetEncoding = self.settings.subscribePacketEncoding
- 
             if isinstance(packet, (bytes, bytearray)) : 
 
                 if self.settings.subscribePacketEncoding : 
@@ -359,12 +358,17 @@ def create( self ):
                         packet = p.getPayloadAsAscii()
 
                     elif packetEncoding == 'utf-8' :
-                        
+
                         binData = binascii.unhexlify( packet )
                         p = Packet(binData)
                         packet = p.getPayloadAsUtf8()
                     else :
-                        packet = record['packetData'].decode('utf-8')
+                        binData = binascii.unhexlify( packet )
+                        p = Packet(binData)
+                        packet = p.getPayloadAsHex()
+
+                if self.settings.subscribeIncludeOriginalPacket :
+                    self.__addValue(View.PACKET_DATA_FULL,record['packetData'].decode('utf-8')) 
 
                 self.__addValue(View.PACKET_DATA, packet)
 
@@ -801,6 +805,12 @@ def create( self ):
 
         elif recordTypeId == definitions.RECORD_MALWARE_EVENT:
             # 125
+
+            for key in record :
+                if isinstance(record[key], (bytes, bytearray)) :
+                     value = record[key].decode('utf-8')
+                     record[key] = value
+
             self.__addValueIfAvailable(
                 View.CLOUD,
                 [ Cache.CLOUDS, record['cloudUuid']] )
@@ -1147,6 +1157,12 @@ def create( self ):
         elif recordTypeId == definitions.RECORD_FILELOG_EVENT or \
              recordTypeId == definitions.RECORD_FILELOG_MALWARE_EVENT:
             # 500 or 502
+
+            for key in record :
+                if isinstance(record[key], (bytes, bytearray)) :
+                     value = record[key].decode('utf-8')
+                     record[key] = value
+
             self.__addValueIfAvailable(
                 View.FILE_POLICY,
                 [ Cache.POLICIES, record['accessControlPolicyUuid']] )
@@ -1229,6 +1245,11 @@ def create( self ):
 
         elif recordTypeId == definitions.METADATA_FILELOG_SHA:
             # 511
+            for key in record :
+                if isinstance(record[key], (bytes, bytearray)) :
+                     value = record[key].decode('utf-8')
+                     record[key] = value
+
             self.__addValueIfAvailable(
                 View.DISPOSITION,
                 [ Cache.FILE_DISPOSITIONS, record['disposition']] )

estreamer/settings/settings.py

@@ -107,6 +107,11 @@ def __init__( self, jsonSettings ):
         else :
             self.subscribePacketEncoding = 'hex'
 
+        if 'includeOriginalPacket' in subscriptionRecords :
+            self.subscribeIncludeOriginalPacket = subscriptionRecords['includeOriginalPacket']
+        else :
+            self.subscribeIncludeOriginalPacket = True
+
         self.subscribePacketData = subscriptionRecords['packetData']
         self.subscribeExtended = subscriptionRecords['extended']
         self.subscribeMetaData = subscriptionRecords['metadata']