fuzz: extend fuzzing coverage (#2205)

example/sha1_fingerprints.csv

@@ -7,6 +7,8 @@
 ################################################################
 #
 # Listingdate,SHA1,Listingreason
+#The first entry (and only the first one) is fake. It is used to trigger the risk in the tests
+1970-01-01 00:00:00,0DDB34F875632C7E1EC09D75827F82D2336DFEB6, Fake
 2022-07-02 07:02:11,3784422d67a6ae0c82d60eeaf3850960a9896cb2,AsyncRAT C&C
 2022-07-01 18:28:10,a0d78b3ddb27cd4f110a0292cab800beadaf9bd2,AsyncRAT C&C
 2022-07-01 09:44:14,4c319ec8b5a66a256a0137d70cd0f9a76bb23fd7,AsyncRAT C&C
@@ -4878,4 +4880,4 @@
 2014-05-04 08:10:31,86d6aade4ba1414a91b1e7fb3cdd7d503692f410,Shylock C&C
 2014-05-04 08:10:26,5afc236d1dd00c9c45457b75226b501b815a59c7,Shylock C&C
 2014-05-04 08:09:56,b08a4939fb88f375a2757eaddc47b1fb8b554439,Shylock C&C
-# END (4871) entries
+# END (4871) entries

fuzz/fuzz_alg_hw_rsi_outliers_da.cpp

@@ -74,6 +74,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   ndpi_predict_linear(values, num_values, predict_periods, &prediction);
 
   /* Data analysis stuff */
+  ndpi_data_print_window_values(a);
   ndpi_data_average(a);
   ndpi_data_mean(a);
   ndpi_data_variance(a);

fuzz/fuzz_alg_hw_rsi_outliers_da.options

@@ -0,0 +1,2 @@
+[libfuzzer]
+close_fd_mask=1

fuzz/fuzz_common_code.c

@@ -42,8 +42,6 @@ void fuzz_init_detection_module(struct ndpi_detection_module_struct **ndpi_info_
 
   if(*ndpi_info_mod == NULL) {
     *ndpi_info_mod = ndpi_init_detection_module(prefs);
-    NDPI_BITMASK_SET_ALL(all);
-    ndpi_set_protocol_detection_bitmask2(*ndpi_info_mod, &all);
 
     NDPI_BITMASK_SET_ALL(debug_bitmask);
     ndpi_set_log_level(*ndpi_info_mod, 4);
@@ -56,6 +54,9 @@ void fuzz_init_detection_module(struct ndpi_detection_module_struct **ndpi_info_
     ndpi_load_malicious_ja3_file(*ndpi_info_mod, "ja3_fingerprints.csv");
     ndpi_load_malicious_sha1_file(*ndpi_info_mod, "sha1_fingerprints.csv");
 
+    NDPI_BITMASK_SET_ALL(all);
+    ndpi_set_protocol_detection_bitmask2(*ndpi_info_mod, &all);
+
     ndpi_finalize_initialization(*ndpi_info_mod);
   }
 }

fuzz/fuzz_gcrypt_aes.cpp

@@ -3,7 +3,7 @@
 #include "fuzzer/FuzzedDataProvider.h"
 
 #define MBEDTLS_CHECK_RETURN_TYPICAL
-#include "../src/lib/third_party/include/gcrypt/aes.h"
+#include "gcrypt/aes.h"
 
 extern int force_no_aesni;
 

fuzz/fuzz_gcrypt_cipher.cpp

@@ -5,8 +5,8 @@
 
 #define MBEDTLS_CHECK_RETURN_TYPICAL
 #define MBEDTLS_INTERNAL_VALIDATE_RET( cond, ret )  do { } while( 0 )
-#include "../src/lib/third_party/include/gcrypt/cipher.h"
-#include "../src/lib/third_party/include/gcrypt/aes.h"
+#include "gcrypt/cipher.h"
+#include "gcrypt/aes.h"
 
 extern int force_no_aesni;
 

fuzz/fuzz_gcrypt_gcm.cpp

@@ -5,9 +5,9 @@
 
 #define MBEDTLS_CHECK_RETURN_TYPICAL
 #define MBEDTLS_INTERNAL_VALIDATE_RET( cond, ret )  do { } while( 0 )
-#include "../src/lib/third_party/include/gcrypt/aes.h"
-#include "../src/lib/third_party/include/gcrypt/cipher.h"
-#include "../src/lib/third_party/include/gcrypt/gcm.h"
+#include "gcrypt/aes.h"
+#include "gcrypt/cipher.h"
+#include "gcrypt/gcm.h"
 
 extern int force_no_aesni;
 

fuzz/fuzz_ndpi_reader.c

@@ -57,9 +57,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
 #endif
 
     workflow = ndpi_workflow_init(prefs, NULL /* pcap handler will be set later */, 0, ndpi_serialization_format_json);
-    // enable all protocols
-    NDPI_BITMASK_SET_ALL(all);
-    ndpi_set_protocol_detection_bitmask2(workflow->ndpi_struct, &all);
 
     NDPI_BITMASK_SET_ALL(debug_bitmask);
     ndpi_set_log_level(workflow->ndpi_struct, 4);
@@ -71,6 +68,10 @@ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
     ndpi_load_malicious_ja3_file(workflow->ndpi_struct, "ja3_fingerprints.csv");
     ndpi_load_malicious_sha1_file(workflow->ndpi_struct, "sha1_fingerprints.csv");
 
+    // enable all protocols
+    NDPI_BITMASK_SET_ALL(all);
+    ndpi_set_protocol_detection_bitmask2(workflow->ndpi_struct, &all);
+
     ndpi_set_detection_preferences(workflow->ndpi_struct, ndpi_pref_enable_tls_block_dissection, 0 /* unused */);
 
     ndpi_set_monitoring_state(workflow->ndpi_struct, NDPI_PROTOCOL_STUN,

src/lib/ndpi_analyze.c

@@ -708,6 +708,7 @@ float ndpi_bin_similarity(struct ndpi_bin *b1, struct ndpi_bin *b2,
 
 /* ********************************************************************************* */
 
+//#define DEBUG_CLUSTER_BINS
 #define MAX_NUM_CLUSTERS  128
 
 /*
@@ -723,7 +724,7 @@ int ndpi_cluster_bins(struct ndpi_bin *bins, u_int16_t num_bins,
 		      u_int8_t num_clusters, u_int16_t *cluster_ids,
 		      struct ndpi_bin *centroids) {
   u_int16_t i, j, max_iterations = 25, num_iterations, num_moves;
-  u_int8_t verbose = 0, alloc_centroids = 0;
+  u_int8_t alloc_centroids = 0;
   char out_buf[256];
   float *bin_score;
   u_int16_t num_cluster_elems[MAX_NUM_CLUSTERS] = { 0 };
@@ -737,8 +738,9 @@ int ndpi_cluster_bins(struct ndpi_bin *bins, u_int16_t num_bins,
   if(num_clusters > num_bins)         num_clusters = num_bins;
   if(num_clusters > MAX_NUM_CLUSTERS) num_clusters = MAX_NUM_CLUSTERS;
 
-  if(verbose)
-    printf("Distributing %u bins over %u clusters\n", num_bins, num_clusters);
+#ifdef DEBUG_CLUSTER_BINS
+  printf("Distributing %u bins over %u clusters\n", num_bins, num_clusters);
+#endif
 
   if((bin_score = (float*)ndpi_calloc(num_bins, sizeof(float))) == NULL)
     return(-2);
@@ -764,10 +766,11 @@ int ndpi_cluster_bins(struct ndpi_bin *bins, u_int16_t num_bins,
 
     cluster_ids[i] = cluster_id;
 
-    if(verbose)
-      printf("Initializing cluster %u for bin %u: %s\n",
-	     cluster_id, i,
-	     ndpi_print_bin(&bins[i], 0, out_buf, sizeof(out_buf)));
+#ifdef DEBUG_CLUSTER_BINS
+    printf("Initializing cluster %u for bin %u: %s\n",
+	   cluster_id, i,
+	   ndpi_print_bin(&bins[i], 0, out_buf, sizeof(out_buf)));
+#endif
 
     num_cluster_elems[cluster_id]++;
   }
@@ -780,12 +783,12 @@ int ndpi_cluster_bins(struct ndpi_bin *bins, u_int16_t num_bins,
     /* Compute the centroids for each cluster */
     memset(bin_score, 0, num_bins*sizeof(float));
 
-    if(verbose) {
-      printf("\nIteration %u\n", num_iterations);
+#ifdef DEBUG_CLUSTER_BINS
+    printf("\nIteration %u\n", num_iterations);
 
-      for(j=0; j<num_clusters; j++)
-	printf("Cluster %u: %u bins\n", j, num_cluster_elems[j]);
-    }
+    for(j=0; j<num_clusters; j++)
+      printf("Cluster %u: %u bins\n", j, num_cluster_elems[j]);
+#endif
 
     for(i=0; i<num_clusters; i++)
       ndpi_reset_bin(&centroids[i]);
@@ -799,9 +802,10 @@ int ndpi_cluster_bins(struct ndpi_bin *bins, u_int16_t num_bins,
     for(i=0; i<num_clusters; i++) {
       ndpi_normalize_bin(&centroids[i]);
 
-      if(verbose)
-	printf("Centroid [%u] %s\n", i,
-	       ndpi_print_bin(&centroids[i], 0, out_buf, sizeof(out_buf)));
+#ifdef DEBUG_CLUSTER_BINS
+      printf("Centroid [%u] %s\n", i,
+	     ndpi_print_bin(&centroids[i], 0, out_buf, sizeof(out_buf)));
+#endif
     }
 
     /* Now let's check if there are bins to move across clusters */
@@ -812,9 +816,10 @@ int ndpi_cluster_bins(struct ndpi_bin *bins, u_int16_t num_bins,
       float best_similarity, current_similarity = 0;
       u_int8_t cluster_id = 0;
 
-      if(verbose)
-	printf("Analysing bin %u [cluster: %u]\n",
-	       i, cluster_ids[i]);
+#ifdef DEBUG_CLUSTER_BINS
+      printf("Analysing bin %u [cluster: %u]\n",
+	     i, cluster_ids[i]);
+#endif
 
 #ifdef COSINE_SIMILARITY
       best_similarity = -1;
@@ -832,8 +837,9 @@ int ndpi_cluster_bins(struct ndpi_bin *bins, u_int16_t num_bins,
 	if(j == cluster_ids[i])
 	  current_similarity = similarity;
 
-	if(verbose)
-	  printf("Bin %u / centroid %u [similarity: %f]\n", i, j, similarity);
+#ifdef DEBUG_CLUSTER_BINS
+	printf("Bin %u / centroid %u [similarity: %f]\n", i, j, similarity);
+#endif
 
 #ifdef COSINE_SIMILARITY
 	if(similarity > best_similarity) {
@@ -857,9 +863,10 @@ int ndpi_cluster_bins(struct ndpi_bin *bins, u_int16_t num_bins,
       bin_score[i] = best_similarity;
 
       if(cluster_ids[i] != cluster_id) {
-	if(verbose)
-	  printf("Moved bin %u from cluster %u -> %u [similarity: %f]\n",
-		 i, cluster_ids[i], cluster_id, best_similarity);
+#ifdef DEBUG_CLUSTER_BINS
+	printf("Moved bin %u from cluster %u -> %u [similarity: %f]\n",
+	       i, cluster_ids[i], cluster_id, best_similarity);
+#endif
 
 	num_cluster_elems[cluster_ids[i]]--;
 	num_cluster_elems[cluster_id]++;
@@ -872,10 +879,10 @@ int ndpi_cluster_bins(struct ndpi_bin *bins, u_int16_t num_bins,
     if(num_moves == 0)
       break;
 
-    if(verbose) {
-      for(j=0; j<num_clusters; j++)
-	printf("Cluster %u: %u bins\n", j, num_cluster_elems[j]);
-    }
+#ifdef DEBUG_CLUSTER_BINS
+    for(j=0; j<num_clusters; j++)
+      printf("Cluster %u: %u bins\n", j, num_cluster_elems[j]);
+#endif
 
 #if 0
     for(j=0; j<num_clusters; j++) {
@@ -1382,7 +1389,6 @@ void ndpi_ses_fitting(double *values, u_int32_t num_values, float *ret_alpha) {
   u_int i;
   float alpha, best_alpha;
   double sse, lowest_sse;
-  int trace = 0;
 
   if(!values || num_values == 0) {
     *ret_alpha = 0;
@@ -1396,8 +1402,9 @@ void ndpi_ses_fitting(double *values, u_int32_t num_values, float *ret_alpha) {
 
     ndpi_ses_init(&ses, alpha, 0.05);
 
-    if(trace)
-      printf("\nDouble Exponential Smoothing [alpha: %.2f]\n", alpha);
+#ifdef SES_DEBUG
+    printf("\nDouble Exponential Smoothing [alpha: %.2f]\n", alpha);
+#endif
 
     sse = 0;
 
@@ -1408,8 +1415,9 @@ void ndpi_ses_fitting(double *values, u_int32_t num_values, float *ret_alpha) {
       if(ndpi_ses_add_value(&ses, values[i], &prediction, &confidence_band) != 0) {
 	diff = fabs(prediction-values[i]);
 
-	if(trace)
-	  printf("%2u)\t%12.3f\t%.3f\t%.3f\n", i, values[i], prediction, diff);
+#ifdef SES_DEBUG
+	printf("%2u)\t%12.3f\t%.3f\t%.3f\n", i, values[i], prediction, diff);
+#endif
 
 	sse += diff*diff;
       }
@@ -1422,13 +1430,15 @@ void ndpi_ses_fitting(double *values, u_int32_t num_values, float *ret_alpha) {
 	lowest_sse = sse, best_alpha = alpha;
     }
 
-    if(trace)
-      printf("[alpha: %.2f] - SSE: %.2f [BEST: alpha: %.2f/SSE: %.2f]\n", alpha, sse,
-	     best_alpha, lowest_sse);
+#ifdef SES_DEBUG
+    printf("[alpha: %.2f] - SSE: %.2f [BEST: alpha: %.2f/SSE: %.2f]\n", alpha, sse,
+	   best_alpha, lowest_sse);
+#endif
   } /* for (alpha) */
 
-  if(trace)
-    printf("BEST [alpha: %.2f][SSE: %.2f]\n", best_alpha, lowest_sse);
+#ifdef SES_DEBUG
+  printf("BEST [alpha: %.2f][SSE: %.2f]\n", best_alpha, lowest_sse);
+#endif
 
   *ret_alpha = best_alpha;
 }
@@ -1528,7 +1538,6 @@ void ndpi_des_fitting(double *values, u_int32_t num_values, float *ret_alpha, fl
   u_int i;
   float alpha, best_alpha, best_beta, beta = 0;
   double sse, lowest_sse;
-  int trace = 0;
 
   if(!values || num_values == 0) {
     *ret_alpha = 0;
@@ -1544,8 +1553,9 @@ void ndpi_des_fitting(double *values, u_int32_t num_values, float *ret_alpha, fl
 
       ndpi_des_init(&des, alpha, beta, 0.05);
 
-      if(trace)
-	printf("\nDouble Exponential Smoothing [alpha: %.2f][beta: %.2f]\n", alpha, beta);
+#ifdef DES_DEBUG
+      printf("\nDouble Exponential Smoothing [alpha: %.2f][beta: %.2f]\n", alpha, beta);
+#endif
 
       sse = 0;
 
@@ -1556,8 +1566,9 @@ void ndpi_des_fitting(double *values, u_int32_t num_values, float *ret_alpha, fl
 	if(ndpi_des_add_value(&des, values[i], &prediction, &confidence_band) != 0) {
 	  diff = fabs(prediction-values[i]);
 
-	  if(trace)
-	    printf("%2u)\t%12.3f\t%.3f\t%.3f\n", i, values[i], prediction, diff);
+#ifdef DES_DEBUG
+	  printf("%2u)\t%12.3f\t%.3f\t%.3f\n", i, values[i], prediction, diff);
+#endif
 
 	  sse += diff*diff;
 	}
@@ -1570,14 +1581,16 @@ void ndpi_des_fitting(double *values, u_int32_t num_values, float *ret_alpha, fl
 	  lowest_sse = sse, best_alpha = alpha, best_beta = beta;
       }
 
-      if(trace)
-	printf("[alpha: %.2f][beta: %.2f] - SSE: %.2f [BEST: alpha: %.2f/beta: %.2f/SSE: %.2f]\n", alpha, beta, sse,
-	       best_alpha, best_beta, lowest_sse);
+#ifdef DES_DEBUG
+      printf("[alpha: %.2f][beta: %.2f] - SSE: %.2f [BEST: alpha: %.2f/beta: %.2f/SSE: %.2f]\n", alpha, beta, sse,
+	     best_alpha, best_beta, lowest_sse);
+#endif
     } /* for (alpha) */
   } /* for (beta) */
 
-  if(trace)
-    printf("BEST [alpha: %.2f][beta: %.2f][SSE: %.2f]\n", best_alpha, best_beta, lowest_sse);
+#ifdef DES_DEBUG
+  printf("BEST [alpha: %.2f][beta: %.2f][SSE: %.2f]\n", best_alpha, best_beta, lowest_sse);
+#endif
 
   *ret_alpha = best_alpha, *ret_beta = best_beta;
 }

src/lib/protocols/tls.c

@@ -1414,9 +1414,6 @@ static void tls_subclassify_by_alpn(struct ndpi_detection_module_struct *ndpi_st
 				    struct ndpi_flow_struct *flow) {
   /* Right now we have only one rule so we can keep it trivial */
 
-  if (!flow->protos.tls_quic.advertised_alpns)
-    return;
-
   if(strlen(flow->protos.tls_quic.advertised_alpns) > NDPI_STATICSTRING_LEN("anydesk/") &&
      strncmp(flow->protos.tls_quic.advertised_alpns, "anydesk/", NDPI_STATICSTRING_LEN("anydesk/")) == 0) {
 #ifdef DEBUG_TLS

tests/cfgs/default/result/tls_malicious_sha1.pcapng.out

@@ -0,0 +1,33 @@
+Guessed flow protos:	0
+
+DPI Packets (TCP):	8	(8.00 pkts/flow)
+Confidence DPI              : 1 (flows)
+Num dissector calls: 1 (1.00 diss/flow)
+LRU cache ookla:      0/0/0 (insert/search/found)
+LRU cache bittorrent: 0/0/0 (insert/search/found)
+LRU cache zoom:       0/0/0 (insert/search/found)
+LRU cache stun:       0/0/0 (insert/search/found)
+LRU cache tls_cert:   0/2/0 (insert/search/found)
+LRU cache mining:     0/0/0 (insert/search/found)
+LRU cache msteams:    0/0/0 (insert/search/found)
+LRU cache stun_zoom:  0/0/0 (insert/search/found)
+Automa host:          2/0 (search/found)
+Automa domain:        2/0 (search/found)
+Automa tls cert:      1/0 (search/found)
+Automa risk mask:     0/0 (search/found)
+Automa common alpns:  2/2 (search/found)
+Patricia risk mask:   0/0 (search/found)
+Patricia risk mask IPv6: 0/0 (search/found)
+Patricia risk:        0/0 (search/found)
+Patricia risk IPv6:   1/0 (search/found)
+Patricia protocols:   0/0 (search/found)
+Patricia protocols IPv6: 1/1 (search/found)
+
+TLS	22	7204	1
+
+JA3 Host Stats: 
+		 IP Address                  	 # JA3C     
+	1	 2001:b07:a3d:c112:9726:f643:a838:b0c4 	 1      
+
+
+	1	TCP [2001:b07:a3d:c112:9726:f643:a838:b0c4]:40294 <-> [2a00:1450:4002:414::2013]:443 [proto: 91/TLS][IP: 126/Google][Encrypted][Confidence: DPI][DPI packets: 8][cat: Web/5][12 pkts/1574 bytes <-> 10 pkts/5630 bytes][Goodput ratio: 34/85][0.12 sec][Hostname/SNI: www.prbtest.dev][(Advertised) ALPNs: h2;http/1.1][(Negotiated) ALPN: h2][bytes ratio: -0.563 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 4/6 23/20 7/7][Pkt Len c2s/s2c min/avg/max/stddev: 86/86 131/563 316/2502 62/920][Risk: ** Malicious SSL Cert/SHA1 Fingerp. **][Risk Score: 50][Risk Info: 0DDB34F875632C7E1EC09D75827F82D2336DFEB6][TLSv1.2][JA3C: 00bcd759cb8ad485fdbf1e7a0c5b94b4][ServerNames: www.prbtest.dev][JA3S: e2bc06b738d7e5d2b0cec5d2196b1d80][Issuer: C=US, O=Google Trust Services LLC, CN=GTS CA 1D4][Subject: CN=www.prbtest.dev][Certificate SHA-1: 0D:DB:34:F8:75:63:2C:7E:1E:C0:9D:75:82:7F:82:D2:33:6D:FE:B6][Firefox][Validity: 2023-11-28 12:50:11 - 2024-02-26 13:39:22][Cipher: TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256][Plen Bins: 16,51,8,0,0,0,0,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,16]