Remove struct ndpi_packet_struct from struct ndpi_flow_struct (#1319

)

There are no real reasons to embed `struct ndpi_packet_struct` (i.e. "packet")
in `struct ndpi_flow_struct` (i.e. "flow"). In other words, we can avoid
saving dissection information of "current packet" into the "flow" state,
i.e. in the flow management table.

The nDPI detection module processes only one packet at the time, so it is
safe to save packet dissection information in `struct ndpi_detection_module_struct`,
reusing always the same "packet" instance and saving a huge amount of memory.
Bottom line: we need only one copy of "packet" (for detection module),
not one for each "flow".

It is not clear how/why "packet" ended up in "flow" in the first place.
It has been there since the beginning of the GIT history, but in the original
OpenDPI code `struct ipoque_packet_struct` was embedded in
`struct ipoque_detection_module_struct`, i.e. there was the same exact
situation this commit wants to achieve.

Most of the changes in this PR are some boilerplate to update something
like "flow->packet" into something like "module->packet" throughout the code.
Some attention has been paid to update `ndpi_init_packet()` since we need
to reset some "packet" fields before starting to process another packet.

There has been one important change, though, in ndpi_detection_giveup().
Nothing changed for the applications/users, but this function can't access
"packet" anymore.
The reason is that this function can be called "asynchronously" with respect
to the data processing, i.e in context where there is no valid notion of
"current packet"; for example ndpiReader calls it after having processed all
the traffic, iterating the entire session table.

Mining LRU stuff seems a bit odd (even before this patch): probably we need
to rethink it, as a follow-up.

python/ndpi.py

@@ -645,6 +645,14 @@
   /* NDPI_PROTOCOL_WIREGUARD */
   uint8_t wireguard_stage;
   uint32_t wireguard_peer_index[2];
+
+  /* NDPI_PROTOCOL_QUIC */
+  u_int8_t *quic_reasm_buf;
+  u_int32_t quic_reasm_buf_len;
+
+  /* NDPI_PROTOCOL_CSGO */
+  uint8_t csgo_strid[18],csgo_state,csgo_s2;
+  uint32_t csgo_id2;
 };
 
 struct ndpi_int_one_line_struct {
@@ -941,6 +949,9 @@
 
   uint8_t direction_detect_disable:1, /* disable internal detection of packet direction */
     _pad:7;
+
+  /* Current packet */
+  struct ndpi_packet_struct packet;
 };
 
 #define NDPI_CIPHER_SAFE                        0
@@ -1163,15 +1174,13 @@
   uint8_t ovpn_session_id[8];
   uint8_t ovpn_counter;
 
+  /* Flow key used to search a match into the mining cache */
+  u_int32_t key_mining_cache;
+
   /* NDPI_PROTOCOL_TINC */
   uint8_t tinc_state;
   struct tinc_cache_entry tinc_cache_entry;
 
-  /* NDPI_PROTOCOL_CSGO */
-  uint8_t csgo_strid[18],csgo_state,csgo_s2;
-  uint32_t csgo_id2;
-  /* internal structures to save functions calls */
-  struct ndpi_packet_struct packet;
   struct ndpi_id_struct *src;
   struct ndpi_id_struct *dst;
 };

python/ndpi_typestruct.py

@@ -276,11 +276,9 @@ class CustomCategories(Structure):
     ("tinc_cache", POINTER(Cache)),
     ("proto_defaults", NDPIProtoDefaultsT * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() +
                                              ndpi.ndpi_wrap_ndpi_max_num_custom_protocols())),
-    ("http_dont_dissect_response", c_uint8, 1),
-    ("dns_dont_dissect_response", c_uint8, 1),
     ("direction_detect_disable", c_uint8, 1),
-    ("disable_metadata_export", c_uint8, 1),
-    ("hyperscan", c_void_p)
+    ('_pad', c_uint8, 7),
+    ('packet', NDPIPacketStruct),
 ]
 
 
@@ -408,6 +406,12 @@ class NDPIFlowUdpStruct(Structure):
         ('memcached_matches', c_uint8),
         ('wireguard_stage', c_uint8),
         ('wireguard_peer_index', c_uint32 * 2),
+        ('quic_reasm_buf', POINTER(c_uint8)),
+        ('quic_reasm_buf_len', c_uint32),
+        ('csgo_strid', c_uint8 * 18),
+        ('csgo_state', c_uint8),
+        ('csgo_s2', c_uint8),
+        ('csgo_id2', c_uint32),
     ]
 
 
@@ -735,15 +739,9 @@ class NDPIFlowStructStack(Structure):
     ('starcraft_udp_stage', c_uint8, 3),
     ('ovpn_session_id', c_uint8 * 8),
     ('ovpn_counter', c_uint8),
+    ('key_mining_cache', c_uint32),
     ('tinc_state', c_uint8),
     ('TincCacheEntry', TincCacheEntry),
-    ('csgo_strid', c_uint8 * 18),
-    ('csgo_state', c_uint8),
-    ('csgo_s2', c_uint8),
-    ('csgo_id2', c_uint32),
-    ('kxun_counter', c_uint16),
-    ('iqiyi_counter', c_uint16),
-    ('packet', NDPIPacketStruct),
     ('src', POINTER(NDPIIdStruct)),
     ('dst', POINTER(NDPIIdStruct))
 ]

src/include/ndpi_typedefs.h

@@ -1175,6 +1175,9 @@ struct ndpi_detection_module_struct {
   MMDB_s mmdb_city, mmdb_as;
   u_int8_t mmdb_city_loaded, mmdb_as_loaded;
 #endif
+
+  /* Current packet */
+  struct ndpi_packet_struct packet;
 };
 
 #endif /* NDPI_LIB_COMPILATION */
@@ -1423,12 +1426,13 @@ struct ndpi_flow_struct {
   u_int8_t ovpn_session_id[8];
   u_int8_t ovpn_counter;
 
+  /* Flow key used to search a match into the mining cache */
+  u_int32_t key_mining_cache;
+
   /* NDPI_PROTOCOL_TINC */
   u_int8_t tinc_state;
   struct tinc_cache_entry tinc_cache_entry;
 
-  /* internal structures to save functions calls */
-  struct ndpi_packet_struct packet;
   struct ndpi_id_struct *src;
   struct ndpi_id_struct *dst;
 };