Skip to content

Commit

Permalink
Improved Tor exit node download and added IPv6 support
Browse files Browse the repository at this point in the history
  • Loading branch information
@lucaderi
lucaderi committed Feb 24, 2025
1 parent 3d0bfc7 commit 050ea7e
Show file tree
Hide file tree
Showing 7 changed files with 7,944 additions and 522 deletions.
8,396 changes: 7,900 additions & 496 deletions src/lib/inc_generated/ndpi_tor_match.c.inc
View file

Large diffs are not rendered by default.

8 changes: 4 additions & 4 deletions tests/cfgs/default/result/dns.pcap.out
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
DPI Packets (TCP): 6 (6.00 pkts/flow)
DPI Packets (TCP): 9 (9.00 pkts/flow)
DPI Packets (UDP): 6 (2.00 pkts/flow)
Confidence DPI : 4 (flows)
Num dissector calls: 13 (3.25 diss/flow)
Expand All @@ -9,8 +9,8 @@ LRU cache tls_cert: 0/0/0 (insert/search/found)
LRU cache mining: 0/0/0 (insert/search/found)
LRU cache msteams: 0/0/0 (insert/search/found)
LRU cache fpc_dns: 1/1/0 (insert/search/found)
Automa host: 8/3 (search/found)
Automa domain: 8/0 (search/found)
Automa host: 7/3 (search/found)
Automa domain: 7/0 (search/found)
Automa tls cert: 0/0 (search/found)
Automa risk mask: 2/0 (search/found)
Automa common alpns: 0/0 (search/found)
Expand All @@ -26,7 +26,7 @@ MDNS 1 371 1

Acceptable 17 3553 4

1 TCP [2001:b07:a3d:c112:b831:a73f:7974:e604]:49774 <-> [2001:b07:a3d:c112::1]:53 [proto: 5/DNS][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 6][cat: Network/14][6 pkts/490 bytes <-> 5 pkts/2156 bytes][Goodput ratio: 7/82][0.01 sec][Hostname/SNI: opentracker.io][45.9.60.30][DNS Id: 0x3d73][bytes ratio: -0.630 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 2/2 6/5 3/2][Pkt Len c2s/s2c min/avg/max/stddev: 74/74 82/431 108/1294 13/481][TCP Fingerprint: 2_64_65535_108f896b6121/Unknown][PLAIN TEXT (opentracker)][Plen Bins: 0,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,33,0,0,0,0,0,0,0,0,0]
1 TCP [2001:b07:a3d:c112:b831:a73f:7974:e604]:49774 <-> [2001:b07:a3d:c112::1]:53 [proto: 5/DNS][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 9][cat: Network/14][6 pkts/490 bytes <-> 5 pkts/2156 bytes][Goodput ratio: 7/82][0.01 sec][Hostname/SNI: opentracker.io][0.0.0.0][DNS Id: 0x3d73][bytes ratio: -0.630 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 2/2 6/5 3/2][Pkt Len c2s/s2c min/avg/max/stddev: 74/74 82/431 108/1294 13/481][Risk: ** Malformed Packet **][Risk Score: 10][Risk Info: Invalid DNS Header][TCP Fingerprint: 2_64_65535_108f896b6121/Unknown][PLAIN TEXT (opentracker)][Plen Bins: 0,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,33,0,0,0,0,0,0,0,0,0]
2 UDP [fe80::a00:27ff:feb3:e62e]:5353 -> [ff02::fb]:5353 [proto: 8/MDNS][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 8/MDNS, Confidence: DPI][DPI packets: 1][cat: Network/14][1 pkts/371 bytes -> 0 pkts/0 bytes][Goodput ratio: 83/0][< 1 sec][Hostname/SNI: e.2.6.e.3.b.e.f.f.f.7.2.0.0.a.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.e.f.ip6.arpa][e.2.6.e.3.b.e.f.f.f.7.2.0.0.a.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.e.f.ip6.arpa][PLAIN TEXT (Android)][Plen Bins: 0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
3 UDP 82.178.113.245:47255 <-> 82.178.158.181:53 [VLAN: 785][proto: 5/DNS][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 5/DNS, Confidence: DPI][DPI packets: 2][cat: Network/14][1 pkts/91 bytes <-> 1 pkts/219 bytes][Goodput ratio: 36/73][0.00 sec][Hostname/SNI: e7.whatsapp.net][169.45.219.235][DNS Id: 0x7843][PLAIN TEXT (whatsapp)][Plen Bins: 0,50,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
4 UDP 192.168.170.20:53 <-> 192.168.170.8:32795 [proto: 5/DNS][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 5/DNS, Confidence: DPI][DPI packets: 3][cat: Network/14][2 pkts/151 bytes <-> 1 pkts/75 bytes][Goodput ratio: 44/43][41.07 sec][Hostname/SNI: www.example.com][0.0.0.0][DNS Id: 0xbc1f][PLAIN TEXT (google)][Plen Bins: 0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
12 changes: 6 additions & 6 deletions tests/cfgs/default/result/dns_lots_of_answers.pcapng.out
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
DPI Packets (TCP): 9 (9.00 pkts/flow)
DPI Packets (TCP): 12 (12.00 pkts/flow)
DPI Packets (UDP): 2 (2.00 pkts/flow)
Confidence DPI : 2 (flows)
Num dissector calls: 2 (1.00 diss/flow)
Expand All @@ -8,9 +8,9 @@ LRU cache stun: 0/0/0 (insert/search/found)
LRU cache tls_cert: 0/0/0 (insert/search/found)
LRU cache mining: 0/0/0 (insert/search/found)
LRU cache msteams: 0/0/0 (insert/search/found)
LRU cache fpc_dns: 2/1/0 (insert/search/found)
Automa host: 4/4 (search/found)
Automa domain: 4/0 (search/found)
LRU cache fpc_dns: 0/1/0 (insert/search/found)
Automa host: 2/2 (search/found)
Automa domain: 2/0 (search/found)
Automa tls cert: 0/0 (search/found)
Automa risk mask: 1/0 (search/found)
Automa common alpns: 0/0 (search/found)
Expand All @@ -25,5 +25,5 @@ DNS 16 2200 2

Acceptable 16 2200 2

1 TCP 192.168.12.169:4026 <-> 192.168.12.1:53 [proto: 5/DNS][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 9][cat: Network/14][8 pkts/586 bytes <-> 6 pkts/1118 bytes][Goodput ratio: 7/62][3.17 sec][Hostname/SNI: bstream.hzmklvdieo.com][169.197.119.239][DNS Id: 0x474c][bytes ratio: -0.312 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/2 443/359 1056/716 375/357][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 73/186 108/764 14/258][TCP Fingerprint: 2_64_65535_685ad951a756/Android][PLAIN TEXT (bstream)][Plen Bins: 0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
2 UDP 192.168.12.156:54660 <-> 192.168.12.1:53 [proto: 5/DNS][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 5/DNS, Confidence: DPI][DPI packets: 2][cat: Network/14][1 pkts/91 bytes <-> 1 pkts/405 bytes][Goodput ratio: 53/89][0.32 sec][Hostname/SNI: dinamicx.alibabausercontent.com][163.181.50.229][DNS Id: 0x0c54][PLAIN TEXT (dinamic)][Plen Bins: 0,50,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
1 TCP 192.168.12.169:4026 <-> 192.168.12.1:53 [proto: 5/DNS][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 12][cat: Network/14][8 pkts/586 bytes <-> 6 pkts/1118 bytes][Goodput ratio: 7/62][3.17 sec][Hostname/SNI: bstream.hzmklvdieo.com][0.0.0.0][DNS Id: 0x474c][bytes ratio: -0.312 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/2 443/359 1056/716 375/357][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 73/186 108/764 14/258][Risk: ** Malformed Packet **][Risk Score: 10][Risk Info: Invalid DNS Header][TCP Fingerprint: 2_64_65535_685ad951a756/Android][PLAIN TEXT (bstream)][Plen Bins: 0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
2 UDP 192.168.12.156:54660 <-> 192.168.12.1:53 [proto: 5/DNS][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 5/DNS, Confidence: DPI][DPI packets: 2][cat: Network/14][1 pkts/91 bytes <-> 1 pkts/405 bytes][Goodput ratio: 53/89][0.32 sec][Hostname/SNI: dinamicx.alibabausercontent.com][0.0.0.0][DNS Id: 0x0c54][Risk: ** Malformed Packet **][Risk Score: 10][Risk Info: Invalid DNS Header][PLAIN TEXT (dinamic)][Plen Bins: 0,50,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
6 changes: 3 additions & 3 deletions tests/cfgs/default/result/fuzz-2006-06-26-2594.pcap.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ LRU cache tls_cert: 0/0/0 (insert/search/found)
LRU cache mining: 0/66/0 (insert/search/found)
LRU cache msteams: 0/0/0 (insert/search/found)
LRU cache fpc_dns: 0/66/0 (insert/search/found)
Automa host: 237/0 (search/found)
Automa domain: 230/0 (search/found)
Automa host: 236/0 (search/found)
Automa domain: 229/0 (search/found)
Automa tls cert: 0/0 (search/found)
Automa risk mask: 124/0 (search/found)
Automa common alpns: 0/0 (search/found)
Expand Down Expand Up @@ -110,7 +110,7 @@ Unrated 33 4066 33
67 UDP 192.168.1.2:2739 <-> 192.168.1.1:53 [proto: 5/DNS][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 5/DNS, Confidence: DPI][DPI packets: 2][cat: Network/14][1 pkts/82 bytes <-> 1 pkts/105 bytes][Goodput ratio: 48/59][0.00 sec][Hostname/SNI: 1.0.0.127.in-addr.arpa][0.0.0.0][DNS Id: 0x6ade][DNS Ptr: localhost][Plen Bins: 0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
68 UDP 192.168.1.2:2743 <-> 192.168.1.1:53 [proto: 5/DNS][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 5/DNS, Confidence: DPI][DPI packets: 2][cat: Network/14][1 pkts/82 bytes <-> 1 pkts/105 bytes][Goodput ratio: 48/59][0.00 sec][Hostname/SNI: 1.0.0.127.in-addr.arpa][0.0.0.0][DNS Id: 0x7cc2][DNS Ptr: local_ost][Plen Bins: 0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
69 UDP 192.168.1.2:2753 <-> 192.168.1.1:53 [proto: 5/DNS][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 5/DNS, Confidence: DPI][DPI packets: 2][cat: Network/14][1 pkts/82 bytes <-> 1 pkts/105 bytes][Goodput ratio: 48/59][0.00 sec][Hostname/SNI: 1.0.0.527.in-addr.arpa][0.0.0.0][DNS Id: 0x48ce][DNS Ptr: locathost][Plen Bins: 0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
70 UDP 192.168.1.2:2755 <-> 192.168.1.1:53 [proto: 5/DNS][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 5/DNS, Confidence: DPI][DPI packets: 2][cat: Network/14][1 pkts/82 bytes <-> 1 pkts/105 bytes][Goodput ratio: 48/59][0.00 sec][Hostname/SNI: 1.0.0.127.in-addr.arpa][0.0.0.0][DNS Id: 0x55f0][DNS Ptr: localhost][Plen Bins: 0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
70 UDP 192.168.1.2:2755 <-> 192.168.1.1:53 [proto: 5/DNS][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 5/DNS, Confidence: DPI][DPI packets: 2][cat: Network/14][1 pkts/82 bytes <-> 1 pkts/105 bytes][Goodput ratio: 48/59][0.00 sec][0.0.0.0][DNS Id: 0x55f0][Risk: ** Malformed Packet **][Risk Score: 10][Risk Info: Invalid DNS Header][Plen Bins: 0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
71 UDP 192.168.1.2:2757 <-> 192.168.1.1:53 [proto: 5/DNS][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 5/DNS, Confidence: DPI][DPI packets: 2][cat: Network/14][1 pkts/82 bytes <-> 1 pkts/105 bytes][Goodput ratio: 48/59][0.00 sec][0.0.0.0][DNS Id: 0xdff0][Risk: ** Malformed Packet **** Non-Printable/Invalid Chars Detected **][Risk Score: 110][Risk Info: Invalid chars detected in domain name / Invalid DNS Query Lenght][Plen Bins: 0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
72 UDP 192.168.1.2:2761 <-> 192.168.1.1:53 [proto: 5/DNS][IP: 0/Unknown][ClearText][Confidence: Match by port][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 2][cat: Network/14][1 pkts/82 bytes <-> 1 pkts/105 bytes][Goodput ratio: 11/59][0.00 sec][0.0.0.0][Plen Bins: 50,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
73 UDP 192.168.1.2:2767 <-> 192.168.1.1:53 [proto: 5/DNS][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 5/DNS, Confidence: DPI][DPI packets: 2][cat: Network/14][1 pkts/82 bytes <-> 1 pkts/105 bytes][Goodput ratio: 48/59][0.00 sec][Hostname/SNI: 1.0.0.127.in-addr.arpa][0.0.0.0][DNS Id: 0x78fd][Plen Bins: 0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
Expand Down
8 changes: 4 additions & 4 deletions tests/cfgs/dns_sub_enable/result/dns.pcap.out
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
DPI Packets (TCP): 6 (6.00 pkts/flow)
DPI Packets (TCP): 9 (9.00 pkts/flow)
DPI Packets (UDP): 6 (2.00 pkts/flow)
Confidence DPI : 4 (flows)
Num dissector calls: 13 (3.25 diss/flow)
Expand All @@ -9,8 +9,8 @@ LRU cache tls_cert: 0/0/0 (insert/search/found)
LRU cache mining: 0/0/0 (insert/search/found)
LRU cache msteams: 0/0/0 (insert/search/found)
LRU cache fpc_dns: 1/1/0 (insert/search/found)
Automa host: 8/3 (search/found)
Automa domain: 8/0 (search/found)
Automa host: 7/3 (search/found)
Automa domain: 7/0 (search/found)
Automa tls cert: 0/0 (search/found)
Automa risk mask: 0/0 (search/found)
Automa common alpns: 0/0 (search/found)
Expand All @@ -28,7 +28,7 @@ WhatsApp 2 310 1

Acceptable 17 3553 4

1 TCP [2001:b07:a3d:c112:b831:a73f:7974:e604]:49774 <-> [2001:b07:a3d:c112::1]:53 [proto: 5/DNS][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 6][cat: Network/14][6 pkts/490 bytes <-> 5 pkts/2156 bytes][Goodput ratio: 7/82][0.01 sec][Hostname/SNI: opentracker.io][45.9.60.30][DNS Id: 0x3d73][bytes ratio: -0.630 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 2/2 6/5 3/2][Pkt Len c2s/s2c min/avg/max/stddev: 74/74 82/431 108/1294 13/481][TCP Fingerprint: 2_64_65535_108f896b6121/Unknown][PLAIN TEXT (opentracker)][Plen Bins: 0,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,33,0,0,0,0,0,0,0,0,0]
1 TCP [2001:b07:a3d:c112:b831:a73f:7974:e604]:49774 <-> [2001:b07:a3d:c112::1]:53 [proto: 5/DNS][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 9][cat: Network/14][6 pkts/490 bytes <-> 5 pkts/2156 bytes][Goodput ratio: 7/82][0.01 sec][Hostname/SNI: opentracker.io][0.0.0.0][DNS Id: 0x3d73][bytes ratio: -0.630 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 2/2 6/5 3/2][Pkt Len c2s/s2c min/avg/max/stddev: 74/74 82/431 108/1294 13/481][Risk: ** Malformed Packet **][Risk Score: 10][Risk Info: Invalid DNS Header][TCP Fingerprint: 2_64_65535_108f896b6121/Unknown][PLAIN TEXT (opentracker)][Plen Bins: 0,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,33,0,0,0,0,0,0,0,0,0]
2 UDP [fe80::a00:27ff:feb3:e62e]:5353 -> [ff02::fb]:5353 [proto: 8/MDNS][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 8/MDNS, Confidence: DPI][DPI packets: 1][cat: Network/14][1 pkts/371 bytes -> 0 pkts/0 bytes][Goodput ratio: 83/0][< 1 sec][Hostname/SNI: e.2.6.e.3.b.e.f.f.f.7.2.0.0.a.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.e.f.ip6.arpa][e.2.6.e.3.b.e.f.f.f.7.2.0.0.a.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.e.f.ip6.arpa][PLAIN TEXT (Android)][Plen Bins: 0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
3 UDP 82.178.113.245:47255 <-> 82.178.158.181:53 [VLAN: 785][proto: 5.142/DNS.WhatsApp][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 5.142/DNS.WhatsApp, Confidence: DPI][DPI packets: 2][cat: Network/14][1 pkts/91 bytes <-> 1 pkts/219 bytes][Goodput ratio: 36/73][0.00 sec][Hostname/SNI: e7.whatsapp.net][169.45.219.235][DNS Id: 0x7843][PLAIN TEXT (whatsapp)][Plen Bins: 0,50,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
4 UDP 192.168.170.20:53 <-> 192.168.170.8:32795 [proto: 5.126/DNS.Google][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 5.126/DNS.Google, Confidence: DPI][DPI packets: 3][cat: Network/14][2 pkts/151 bytes <-> 1 pkts/75 bytes][Goodput ratio: 44/43][41.07 sec][Hostname/SNI: www.example.com][0.0.0.0][DNS Id: 0xbc1f][PLAIN TEXT (google)][Plen Bins: 0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
8 changes: 4 additions & 4 deletions tests/cfgs/subclassification_disable/result/dns.pcap.out
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
DPI Packets (TCP): 6 (6.00 pkts/flow)
DPI Packets (TCP): 9 (9.00 pkts/flow)
DPI Packets (UDP): 6 (2.00 pkts/flow)
Confidence DPI : 4 (flows)
Num dissector calls: 13 (3.25 diss/flow)
Expand All @@ -9,8 +9,8 @@ LRU cache tls_cert: 0/0/0 (insert/search/found)
LRU cache mining: 0/0/0 (insert/search/found)
LRU cache msteams: 0/0/0 (insert/search/found)
LRU cache fpc_dns: 1/1/0 (insert/search/found)
Automa host: 8/3 (search/found)
Automa domain: 8/0 (search/found)
Automa host: 7/3 (search/found)
Automa domain: 7/0 (search/found)
Automa tls cert: 0/0 (search/found)
Automa risk mask: 2/0 (search/found)
Automa common alpns: 0/0 (search/found)
Expand All @@ -26,7 +26,7 @@ MDNS 1 371 1

Acceptable 17 3553 4

1 TCP [2001:b07:a3d:c112:b831:a73f:7974:e604]:49774 <-> [2001:b07:a3d:c112::1]:53 [proto: 5/DNS][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 6][cat: Network/14][6 pkts/490 bytes <-> 5 pkts/2156 bytes][Goodput ratio: 7/82][0.01 sec][Hostname/SNI: opentracker.io][45.9.60.30][DNS Id: 0x3d73][bytes ratio: -0.630 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 2/2 6/5 3/2][Pkt Len c2s/s2c min/avg/max/stddev: 74/74 82/431 108/1294 13/481][TCP Fingerprint: 2_64_65535_108f896b6121/Unknown][PLAIN TEXT (opentracker)][Plen Bins: 0,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,33,0,0,0,0,0,0,0,0,0]
1 TCP [2001:b07:a3d:c112:b831:a73f:7974:e604]:49774 <-> [2001:b07:a3d:c112::1]:53 [proto: 5/DNS][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 9][cat: Network/14][6 pkts/490 bytes <-> 5 pkts/2156 bytes][Goodput ratio: 7/82][0.01 sec][Hostname/SNI: opentracker.io][0.0.0.0][DNS Id: 0x3d73][bytes ratio: -0.630 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 2/2 6/5 3/2][Pkt Len c2s/s2c min/avg/max/stddev: 74/74 82/431 108/1294 13/481][Risk: ** Malformed Packet **][Risk Score: 10][Risk Info: Invalid DNS Header][TCP Fingerprint: 2_64_65535_108f896b6121/Unknown][PLAIN TEXT (opentracker)][Plen Bins: 0,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,33,0,0,0,0,0,0,0,0,0]
2 UDP [fe80::a00:27ff:feb3:e62e]:5353 -> [ff02::fb]:5353 [proto: 8/MDNS][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 8/MDNS, Confidence: DPI][DPI packets: 1][cat: Network/14][1 pkts/371 bytes -> 0 pkts/0 bytes][Goodput ratio: 83/0][< 1 sec][Hostname/SNI: e.2.6.e.3.b.e.f.f.f.7.2.0.0.a.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.e.f.ip6.arpa][e.2.6.e.3.b.e.f.f.f.7.2.0.0.a.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.e.f.ip6.arpa][PLAIN TEXT (Android)][Plen Bins: 0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
3 UDP 82.178.113.245:47255 <-> 82.178.158.181:53 [VLAN: 785][proto: 5/DNS][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 5/DNS, Confidence: DPI][DPI packets: 2][cat: Network/14][1 pkts/91 bytes <-> 1 pkts/219 bytes][Goodput ratio: 36/73][0.00 sec][Hostname/SNI: e7.whatsapp.net][169.45.219.235][DNS Id: 0x7843][PLAIN TEXT (whatsapp)][Plen Bins: 0,50,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
4 UDP 192.168.170.20:53 <-> 192.168.170.8:32795 [proto: 5/DNS][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 5/DNS, Confidence: DPI][DPI packets: 3][cat: Network/14][2 pkts/151 bytes <-> 1 pkts/75 bytes][Goodput ratio: 44/43][41.07 sec][Hostname/SNI: www.example.com][0.0.0.0][DNS Id: 0xbc1f][PLAIN TEXT (google)][Plen Bins: 0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
28 changes: 23 additions & 5 deletions utils/tor_ip_addresses_download.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,26 +7,44 @@ cd "$(dirname "${0}")" || exit 1
DEST=../src/lib/inc_generated/ndpi_tor_match.c.inc
LIST=/tmp/tor.list
LIST_MERGED=/tmp/tor.list_m
LIST6_MERGED=/tmp/tor.list_m6
LIST_MERGED_U=/tmp/tor.list_m_u
LIST6_MERGED_U=/tmp/tor.list_m6_u
# There are at least two lists:
# * https://torstatus.rueckgr.at/ip_list_all.php/Tor_ip_list_ALL.csv
# * https://check.torproject.org/torbulkexitlist
# The latter seems to be more "stable" (the former changes every few seconds!)
ORIGIN="https://check.torproject.org/torbulkexitlist"


echo "(1) Downloading file... ${ORIGIN}"
http_response=$(curl -s -o $LIST -w "%{http_code}" ${ORIGIN})
check_http_response "${http_response}"
is_file_empty "${LIST}"
./mergeipaddrlist.py $LIST > $LIST_MERGED
is_file_empty "${LIST_MERGED}"

#TODO: TOR relays don't support ipv6 yet
ORIGIN="https://raw.githubusercontent.com/alireza-rezaee/tor-nodes/refs/heads/main/latest.all.csv"
echo "(2) Downloading file... ${ORIGIN}"
http_response=$(curl -s -o $LIST -w "%{http_code}" ${ORIGIN})
check_http_response "${http_response}"
is_file_empty "${LIST}"

cat "${LIST}" | cut -d ',' -f 2 | grep '\.' | tr -d '[:blank:]' > "${LIST}.ipv4"
cat "${LIST}" | cut -d ',' -f 2 | grep ':' | tr -d '[:blank:]' > "${LIST}.ipv6"

./mergeipaddrlist.py ${LIST}.ipv4 >> $LIST_MERGED
is_file_empty "${LIST_MERGED}"

./mergeipaddrlist.py ${LIST}.ipv6 > $LIST6_MERGED
is_file_empty "${LIST6_MERGED}"

echo "(2) Processing IP addresses..."
./ipaddr2list.py $LIST_MERGED NDPI_PROTOCOL_TOR > $DEST
rm -f "${LIST}" "${LIST_MERGED}"
sort -u $LIST_MERGED > $LIST_MERGED_U
sort -u $LIST6_MERGED > $LIST6_MERGED_U

./ipaddr2list.py $LIST_MERGED_U NDPI_PROTOCOL_TOR $LIST6_MERGED_U > $DEST
is_file_empty "${DEST}"

#rm -f ${TMP} ${LIST} ${LIST6} ${LIST_MERGED} ${LIST6_MERGED} ${LIST_MERGED_U} ${LIST6_MERGED_U}

echo "(3) TOR IPs are available in $DEST"
exit 0

0 comments on commit 050ea7e

Please sign in to comment.