diff --git a/include/ArpStatsHashMatrix.h b/include/ArpStatsHashMatrix.h index c936f85ba3cf..e2a3d9e72ab8 100644 --- a/include/ArpStatsHashMatrix.h +++ b/include/ArpStatsHashMatrix.h @@ -29,7 +29,9 @@ class ArpStatsHashMatrix : public GenericHash { public: ArpStatsHashMatrix(NetworkInterface *iface, u_int _num_hashes, u_int _max_hash_size); - ArpStatsMatrixElement* get(const u_int8_t _src_mac[6], const u_int8_t _dst_mac[6], bool * const src2dst); + ArpStatsMatrixElement* get(const u_int8_t _src_mac[6], + const u_int32_t _src_ip, const u_int32_t _dst_ip, + bool * const src2dst); void lua(lua_State* vm); }; diff --git a/include/ArpStatsMatrixElement.h b/include/ArpStatsMatrixElement.h index 8a1f2f9e62e0..0354179722ec 100644 --- a/include/ArpStatsMatrixElement.h +++ b/include/ArpStatsMatrixElement.h @@ -33,27 +33,32 @@ class ArpStatsMatrixElement : public GenericHashEntry { } src2dst, dst2src; } stats; - u_int8_t src_mac[6]; - u_int8_t dst_mac[6]; + u_int8_t src_mac[6], dst_mac[6]; + u_int32_t src_ip, dst_ip; public: - ArpStatsMatrixElement(NetworkInterface *_iface, const u_int8_t _src_mac[6], const u_int8_t _dst_mac[6], bool * const src2dst); + ArpStatsMatrixElement(NetworkInterface *_iface, + const u_int8_t _src_mac[6], const u_int8_t _dst_mac[6], + const u_int32_t _src_ip, const u_int32_t _dst_ip); ~ArpStatsMatrixElement(); inline void incArpReplies(bool src2dst) { src2dst ? stats.src2dst.replies++ : stats.dst2src.replies++; updateSeen(); } + inline void incArpRequests(bool src2dst) { src2dst ? stats.src2dst.requests++ : stats.dst2src.requests++; updateSeen(); } - bool equal(const u_int8_t _src_mac[6], const u_int8_t _dst_mac[6], bool * const src2dst) const; + bool equal(const u_int8_t _src_mac[6], + const u_int32_t _src_ip, const u_int32_t _dst_ip, + bool * const src2dst) const; virtual bool idle(); u_int32_t key(); void lua(lua_State* vm); - void print() const; + void print(char *msg) const; }; #endif /* _ARP_STATS_MATRIX_ELEMENT_H_ */ diff --git a/include/DummyInterface.h b/include/DummyInterface.h index d29d97e632f1..3745f684660b 100644 --- a/include/DummyInterface.h +++ b/include/DummyInterface.h @@ -26,7 +26,7 @@ #ifndef HAVE_NEDGE -class DummyInterface : public ParserInterface { +class DummyInterface : public ZMQParserInterface { private: inline u_int32_t getNumDroppedPackets() { return(0); }; diff --git a/include/Flow.h b/include/Flow.h index 66fc276fcafb..d86fd6cd2da6 100644 --- a/include/Flow.h +++ b/include/Flow.h @@ -184,18 +184,14 @@ class Flow : public GenericHashEntry { void updatePacketStats(InterarrivalStats *stats, const struct timeval *when); void dumpPacketStats(lua_State* vm, bool cli2srv_direction); bool isReadyToPurge(); - inline bool isBlacklistedFlow() { - return(cli_host && srv_host && (cli_host->isBlacklisted() - || srv_host->isBlacklisted() - || (get_protocol_category() == CUSTOM_CATEGORY_MALWARE))); - }; + bool isBlacklistedFlow() const; inline bool isDeviceAllowedProtocol() { return(!cli_host || !srv_host || ((cli_host->getDeviceAllowedProtocolStatus(ndpiDetectedProtocol, true) == device_proto_allowed) && (srv_host->getDeviceAllowedProtocolStatus(ndpiDetectedProtocol, false) == device_proto_allowed))); } char* printTCPflags(u_int8_t flags, char * const buf, u_int buf_len) const; - inline bool isProto(u_int16_t p ) { return((ndpi_get_lower_proto(ndpiDetectedProtocol) == p) ? true : false); } + inline bool isProto(u_int16_t p ) const { return((ndpi_get_lower_proto(ndpiDetectedProtocol) == p) ? true : false); } #ifdef NTOPNG_PRO void update_pools_stats(const struct timeval *tv, u_int64_t diff_sent_packets, u_int64_t diff_sent_bytes, @@ -222,12 +218,12 @@ class Flow : public GenericHashEntry { struct site_categories* getFlowCategory(bool force_categorization); void freeDPIMemory(); bool isTiny(); - inline bool isSSL() { return(isProto(NDPI_PROTOCOL_SSL)); } - inline bool isSSH() { return(isProto(NDPI_PROTOCOL_SSH)); } - inline bool isDNS() { return(isProto(NDPI_PROTOCOL_DNS)); } - inline bool isDHCP() { return(isProto(NDPI_PROTOCOL_DHCP)); } - inline bool isHTTP() { return(isProto(NDPI_PROTOCOL_HTTP)); } - inline bool isICMP() { return(isProto(NDPI_PROTOCOL_IP_ICMP) || isProto(NDPI_PROTOCOL_IP_ICMPV6)); } + inline bool isSSL() const { return(isProto(NDPI_PROTOCOL_SSL)); } + inline bool isSSH() const { return(isProto(NDPI_PROTOCOL_SSH)); } + inline bool isDNS() const { return(isProto(NDPI_PROTOCOL_DNS)); } + inline bool isDHCP() const { return(isProto(NDPI_PROTOCOL_DHCP)); } + inline bool isHTTP() const { return(isProto(NDPI_PROTOCOL_HTTP)); } + inline bool isICMP() const { return(isProto(NDPI_PROTOCOL_IP_ICMP) || isProto(NDPI_PROTOCOL_IP_ICMPV6)); } inline bool isMaskedFlow() { return(!get_cli_host() || Utils::maskHost(get_cli_host()->isLocalHost()) || !get_srv_host() || Utils::maskHost(get_srv_host()->isLocalHost())); @@ -298,7 +294,7 @@ class Flow : public GenericHashEntry { void addFlowStats(bool cli2srv_direction, u_int in_pkts, u_int in_bytes, u_int in_goodput_bytes, u_int out_pkts, u_int out_bytes, u_int out_goodput_bytes, time_t last_seen); inline bool isThreeWayHandshakeOK() { return(twh_ok); }; - inline bool isDetectionCompleted() { return(detection_completed); }; + inline bool isDetectionCompleted() const { return(detection_completed); }; inline struct ndpi_flow_struct* get_ndpi_flow() { return(ndpiFlow); }; inline void* get_cli_id() { return(cli_id); }; inline void* get_srv_id() { return(srv_id); }; @@ -330,26 +326,26 @@ class Flow : public GenericHashEntry { inline time_t get_partial_first_seen() { return(last_db_dump.last_dump == 0 ? get_first_seen() : last_db_dump.last_dump); }; inline time_t get_partial_last_seen() { return(get_last_seen()); }; inline u_int32_t get_duration() { return((u_int32_t)(get_last_seen()-get_first_seen())); }; - inline char* get_protocol_name() { return(Utils::l4proto2name(protocol)); }; - inline ndpi_protocol get_detected_protocol() { return(isDetectionCompleted() ? ndpiDetectedProtocol : ndpiUnknownProtocol); }; + inline char* get_protocol_name() const { return(Utils::l4proto2name(protocol)); }; + inline ndpi_protocol get_detected_protocol() const { return(isDetectionCompleted() ? ndpiDetectedProtocol : ndpiUnknownProtocol); }; inline Host* get_cli_host() { return(cli_host); }; inline Host* get_srv_host() { return(srv_host); }; inline char* get_json_info() { return(json_info); }; - inline ndpi_protocol_breed_t get_protocol_breed() { + inline ndpi_protocol_breed_t get_protocol_breed() const { return(ndpi_get_proto_breed(iface->get_ndpi_struct(), isDetectionCompleted() ? ndpiDetectedProtocol.app_protocol : NDPI_PROTOCOL_UNKNOWN)); }; - inline const char * const get_protocol_breed_name() { + inline const char * const get_protocol_breed_name() const { return(ndpi_get_proto_breed_name(iface->get_ndpi_struct(), get_protocol_breed())); }; - inline ndpi_protocol_category_t get_protocol_category() { + inline ndpi_protocol_category_t get_protocol_category() const { return(ndpi_get_proto_category(iface->get_ndpi_struct(), isDetectionCompleted() ? ndpiDetectedProtocol : ndpiUnknownProtocol)); }; - inline const char * const get_protocol_category_name() { + inline const char * const get_protocol_category_name() const { return(ndpi_category_get_name(iface->get_ndpi_struct(), get_protocol_category())); }; - char* get_detected_protocol_name(char *buf, u_int buf_len) { + char* get_detected_protocol_name(char *buf, u_int buf_len) const { return(ndpi_protocol2name(iface->get_ndpi_struct(), isDetectionCompleted() ? ndpiDetectedProtocol : ndpiUnknownProtocol, buf, buf_len)); @@ -366,8 +362,7 @@ class Flow : public GenericHashEntry { u_int64_t get_current_packets_cli2srv(); u_int64_t get_current_packets_srv2cli(); inline bool idle() { return(is_ready_to_be_purged()); } - inline bool is_l7_protocol_guessed() { return(l7_protocol_guessed); }; - char* print(char *buf, u_int buf_len); + char* print(char *buf, u_int buf_len) const; void update_hosts_stats(struct timeval *tv, bool dump_alert); u_int32_t key(); static u_int32_t key(Host *cli, u_int16_t cli_port, @@ -382,7 +377,6 @@ class Flow : public GenericHashEntry { bool *src2srv_direction); bool clientLessThanServer() const; void sumStats(nDPIStats *stats); - void guessProtocol(); bool dumpFlow(bool dump_alert); bool match(AddressTree *ptree); void dissectHTTP(bool src2dst_direction, char *payload, u_int16_t payload_len); diff --git a/include/Host.h b/include/Host.h index 22edcf143219..b39814820d3b 100644 --- a/include/Host.h +++ b/include/Host.h @@ -238,6 +238,7 @@ class Host : public GenericHashEntry { virtual void incNumDNSResponsesSent(u_int32_t ret_code) { }; virtual void incNumDNSResponsesRcvd(u_int32_t ret_code) { }; virtual void luaDNS(lua_State *vm) const { }; + virtual void luaTCP(lua_State *vm) const { }; virtual void postHashAdd(); virtual NetworkStats* getNetworkStats(int16_t networkId) { return(NULL); }; diff --git a/include/HostStats.h b/include/HostStats.h index 274bba8a6af0..2f5c7262343b 100644 --- a/include/HostStats.h +++ b/include/HostStats.h @@ -76,7 +76,7 @@ class HostStats: public Checkpointable, public TimeseriesStats { virtual void decNumFlows(bool as_client, Host *peer) {}; virtual bool hasAnomalies(time_t when) { return false; }; virtual void luaAnomalies(lua_State* vm, time_t when) {}; - virtual void lua(lua_State* vm, bool mask_host, bool host_details, bool verbose, bool tsLua = false); + virtual void lua(lua_State* vm, bool mask_host, DetailsLevel details_level, bool tsLua = false); #ifdef NTOPNG_PRO inline void incQuotaEnforcementStats(time_t when, u_int16_t ndpi_proto, diff --git a/include/LocalHost.h b/include/LocalHost.h index 51d20c073b18..954637bdd824 100644 --- a/include/LocalHost.h +++ b/include/LocalHost.h @@ -73,6 +73,7 @@ class LocalHost : public Host { virtual void luaDNS(lua_State *vm) const { stats->luaDNS(vm,false); }; virtual void incrVisitedWebSite(char *hostname) { stats->incrVisitedWebSite(hostname); }; virtual HTTPstats* getHTTPstats() { return(stats->getHTTPstats()); }; + virtual void luaTCP(lua_State *vm) const { stats->lua(vm,false,details_normal); }; virtual void lua(lua_State* vm, AddressTree * ptree, bool host_details, bool verbose, bool returnHost, bool asListElement); diff --git a/include/LocalHostStats.h b/include/LocalHostStats.h index d0c13df39f10..902c969bd18c 100644 --- a/include/LocalHostStats.h +++ b/include/LocalHostStats.h @@ -43,7 +43,7 @@ class LocalHostStats: public HostStats { virtual void updateStats(struct timeval *tv); virtual void getJSONObject(json_object *my_object, DetailsLevel details_level); virtual void deserialize(json_object *obj); - virtual void lua(lua_State* vm, bool mask_host, bool host_details, bool verbose, bool tsLua = false); + virtual void lua(lua_State* vm, bool mask_host, DetailsLevel details_level, bool tsLua = false); virtual void incNumFlows(bool as_client, Host *peer); virtual void decNumFlows(bool as_client, Host *peer); diff --git a/include/NetworkInterface.h b/include/NetworkInterface.h index 72cf5af1cdcc..40e421f22c6f 100644 --- a/include/NetworkInterface.h +++ b/include/NetworkInterface.h @@ -420,7 +420,7 @@ class NetworkInterface : public Checkpointable { const u_char *packet, u_int16_t *ndpiProtocol, Host **srcHost, Host **dstHost, Flow **flow); - void processFlow(ZMQ_Flow *zflow); + void processFlow(Parsed_Flow *zflow); void processInterfaceStats(sFlowInterfaceStats *stats); void getnDPIStats(nDPIStats *stats, AddressTree *allowed_hosts, const char *host_ip, u_int16_t vlan_id); void periodicStatsUpdate(); @@ -523,7 +523,9 @@ class NetworkInterface : public Checkpointable { void runHousekeepingTasks(); void runShutdownTasks(); - ArpStatsMatrixElement* getArpHashMatrixElement(const u_int8_t _src_mac[6], const u_int8_t _dst_mac[6], bool * const src2dst); + ArpStatsMatrixElement* getArpHashMatrixElement(const u_int8_t _src_mac[6], const u_int8_t _dst_mac[6], + const u_int32_t _src_ip, const u_int32_t _dst_ip, + bool * const src2dst); Vlan* getVlan(u_int16_t vlanId, bool createIfNotPresent); AutonomousSystem *getAS(IpAddress *ipa, bool createIfNotPresent); Country* getCountry(const char *country_name, bool createIfNotPresent); diff --git a/include/ParserInterface.h b/include/ParserInterface.h index 8b3bb51256f3..639d18b35b4f 100755 --- a/include/ParserInterface.h +++ b/include/ParserInterface.h @@ -26,43 +26,10 @@ class ParserInterface : public NetworkInterface { private: - typedef std::pair pen_value_t; - typedef std::map labels_map_t; - labels_map_t labels_map; - bool once; - u_int64_t zmq_initial_bytes, zmq_initial_pkts, - zmq_remote_initial_exported_flows; - ZMQ_RemoteStats *zmq_remote_stats, *zmq_remote_stats_shadow; -#ifdef NTOPNG_PRO - CustomAppMaps *custom_app_maps; -#endif - bool getKeyId(char *sym, u_int32_t * const pen, u_int32_t * const field) const; - void addMapping(const char *sym, u_int32_t num, u_int32_t pen = 0); - bool parsePENZeroField(ZMQ_Flow * const flow, u_int32_t field, const char * const value) const; - bool parsePENNtopField(ZMQ_Flow * const flow, u_int32_t field, const char * const value) const; - void parseSingleFlow(json_object *o, u_int8_t source_id, NetworkInterface *iface); - void setFieldMap(const ZMQ_FieldMap * const field_map) const; - void setFieldValueMap(const ZMQ_FieldValueMap * const field_value_map) const; - - u_int8_t parseOptionFieldMap(json_object * const jo) const; - u_int8_t parseOptionFieldValueMap(json_object * const jo) const; public: ParserInterface(const char *endpoint, const char *custom_interface_type = NULL); ~ParserInterface(); - - u_int8_t parseFlow(const char * const payload, int payload_size, u_int8_t source_id, void *data); - u_int8_t parseEvent(const char * const payload, int payload_size, u_int8_t source_id, void *data); - u_int8_t parseCounter(const char * const payload, int payload_size, u_int8_t source_id, void *data); - u_int8_t parseTemplate(const char * const payload, int payload_size, u_int8_t source_id, void *data); - u_int8_t parseOption(const char * const payload, int payload_size, u_int8_t source_id, void *data); - - virtual void setRemoteStats(ZMQ_RemoteStats *zrs); -#ifdef NTOPNG_PRO - virtual bool getCustomAppDetails(u_int32_t remapped_app_id, u_int32_t *const pen, u_int32_t *const app_field, u_int32_t *const app_id); -#endif - u_int32_t getNumDroppedPackets() { return zmq_remote_stats ? zmq_remote_stats->sflow_pkt_sample_drops : 0; }; - virtual void lua(lua_State* vm); }; #endif /* _PARSER_INTERFACE_H_ */ diff --git a/include/SyslogCollectorInterface.h b/include/SyslogCollectorInterface.h new file mode 100644 index 000000000000..c86f2a97574f --- /dev/null +++ b/include/SyslogCollectorInterface.h @@ -0,0 +1,75 @@ +/* + * + * (C) 2019 - ntop.org + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + */ + +#ifndef _SYSLOG_COLLECTOR_INTERFACE_H_ +#define _SYSLOG_COLLECTOR_INTERFACE_H_ + +#include "ntop_includes.h" + +#ifndef HAVE_NEDGE + +class LuaEngine; + +typedef struct { + int socket; + struct sockaddr_in address; +} syslog_client; + +class SyslogCollectorInterface : public SyslogParserInterface { + private: + char *endpoint; + struct sockaddr_in listen_addr; + int listen_sock; + syslog_client connections[MAX_ZMQ_SUBSCRIBERS]; + + struct { + u_int32_t num_flows; + } recvStats; + + public: + SyslogCollectorInterface(const char *_endpoint); + ~SyslogCollectorInterface(); + + int initFDSets(fd_set *read_fds, fd_set *write_fds, fd_set *except_fds); + int handleNewConnection(); + char *clientAddr2Str(syslog_client *client, char *buff); + void closeConnection(syslog_client *client); + int receiveFromClient(syslog_client *client); + + inline const char* get_type() { return(CONST_INTERFACE_TYPE_SYSLOG); }; + inline InterfaceType getIfType() { return(interface_type_SYSLOG); } + inline bool is_ndpi_enabled() { return(false); }; + inline char* getEndpoint(u_int8_t id) { return(endpoint); }; + inline bool isPacketInterface() { return(false); }; + void collect_flows(); + + virtual void purgeIdle(time_t when); + + void startPacketPolling(); + void shutdown(); + bool set_packet_filter(char *filter); + virtual void lua(lua_State* vm); +}; + +#endif /* HAVE_NEDGE */ + +#endif /* _SYSLOG_COLLECTOR_INTERFACE_H_ */ + diff --git a/include/SyslogParserInterface.h b/include/SyslogParserInterface.h new file mode 100644 index 000000000000..32de33051ecf --- /dev/null +++ b/include/SyslogParserInterface.h @@ -0,0 +1,42 @@ +/* + * + * (C) 2019 - ntop.org + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + */ + +#ifndef _SYSLOG_PARSER_INTERFACE_H_ +#define _SYSLOG_PARSER_INTERFACE_H_ + +#include "ntop_includes.h" + +class SyslogParserInterface : public ParserInterface { + private: + + public: + SyslogParserInterface(const char *endpoint, const char *custom_interface_type = NULL); + ~SyslogParserInterface(); + + u_int8_t parseLog(char *log_line, void *data); + + u_int32_t getNumDroppedPackets() { return 0; }; + virtual void lua(lua_State* vm); +}; + +#endif /* _SYSLOG_PARSER_INTERFACE_H_ */ + + diff --git a/include/Utils.h b/include/Utils.h index f4e917ae12d3..92ceedf3bbc9 100755 --- a/include/Utils.h +++ b/include/Utils.h @@ -42,6 +42,7 @@ class Utils { static char* formatTraffic(float numBits, bool bits, char *buf, u_int buf_len); static char* formatPackets(float numPkts, char *buf, u_int buf_len); static char* l4proto2name(u_int8_t proto); + static u_int8_t l4name2proto(char *name); static bool isIPAddress(char *name); static int setThreadAffinity(pthread_t thread, int core_id); static void setThreadName(const char *name); @@ -123,6 +124,7 @@ class Utils { static bool mg_write_retry(struct mg_connection *conn, u_char *b, int len); static bool parseAuthenticatorJson(HTTPAuthenticator *auth, char *content); static void freeAuthenticator(HTTPAuthenticator *auth); + static DetailsLevel bool2DetailsLevel(bool max, bool higher,bool normal = false); /* Patricia Tree */ static patricia_node_t* add_to_ptree(patricia_tree_t *tree, int family, void *addr, int bits); diff --git a/include/ZCCollectorInterface.h b/include/ZCCollectorInterface.h index 5c79fdb244a0..b6d83f52f814 100644 --- a/include/ZCCollectorInterface.h +++ b/include/ZCCollectorInterface.h @@ -26,7 +26,7 @@ #if defined(HAVE_PF_RING) && (!defined(NTOPNG_EMBEDDED_EDITION)) -class ZCCollectorInterface : public ParserInterface { +class ZCCollectorInterface : public ZMQParserInterface { private: u_int32_t cluster_id, queue_id; u_int32_t num_drops; diff --git a/include/CollectorInterface.h b/include/ZMQCollectorInterface.h similarity index 87% rename from include/CollectorInterface.h rename to include/ZMQCollectorInterface.h index b04d2adf1635..7e99129c6f19 100644 --- a/include/CollectorInterface.h +++ b/include/ZMQCollectorInterface.h @@ -19,8 +19,8 @@ * */ -#ifndef _COLLECTOR_INTERFACE_H_ -#define _COLLECTOR_INTERFACE_H_ +#ifndef _ZMQ_COLLECTOR_INTERFACE_H_ +#define _ZMQ_COLLECTOR_INTERFACE_H_ #include "ntop_includes.h" @@ -33,7 +33,7 @@ typedef struct { void *socket; } zmq_subscriber; -class CollectorInterface : public ParserInterface { +class ZMQCollectorInterface : public ZMQParserInterface { private: void *context; struct { @@ -46,8 +46,8 @@ class CollectorInterface : public ParserInterface { zmq_subscriber subscriber[MAX_ZMQ_SUBSCRIBERS]; public: - CollectorInterface(const char *_endpoint); - ~CollectorInterface(); + ZMQCollectorInterface(const char *_endpoint); + ~ZMQCollectorInterface(); inline const char* get_type() { return(CONST_INTERFACE_TYPE_ZMQ); }; inline InterfaceType getIfType() { return(interface_type_ZMQ); } @@ -67,5 +67,5 @@ class CollectorInterface : public ParserInterface { #endif /* HAVE_NEDGE */ -#endif /* _COLLECTOR_INTERFACE_H_ */ +#endif /* _ZMQ_COLLECTOR_INTERFACE_H_ */ diff --git a/include/ZMQParserInterface.h b/include/ZMQParserInterface.h new file mode 100755 index 000000000000..09bd5c815b2a --- /dev/null +++ b/include/ZMQParserInterface.h @@ -0,0 +1,70 @@ +/* + * + * (C) 2013-19 - ntop.org + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + */ + +#ifndef _ZMQ_PARSER_INTERFACE_H_ +#define _ZMQ_PARSER_INTERFACE_H_ + +#include "ntop_includes.h" + +class ZMQParserInterface : public ParserInterface { + private: + typedef std::pair pen_value_t; + typedef std::map labels_map_t; + labels_map_t labels_map; + bool once; + u_int64_t zmq_initial_bytes, zmq_initial_pkts, + zmq_remote_initial_exported_flows; + ZMQ_RemoteStats *zmq_remote_stats, *zmq_remote_stats_shadow; +#ifdef NTOPNG_PRO + CustomAppMaps *custom_app_maps; +#endif + bool getKeyId(char *sym, u_int32_t * const pen, u_int32_t * const field) const; + void addMapping(const char *sym, u_int32_t num, u_int32_t pen = 0); + bool parsePENZeroField(Parsed_Flow * const flow, u_int32_t field, const char * const value) const; + bool parsePENNtopField(Parsed_Flow * const flow, u_int32_t field, const char * const value) const; + void parseSingleFlow(json_object *o, u_int8_t source_id, NetworkInterface *iface); + void setFieldMap(const ZMQ_FieldMap * const field_map) const; + void setFieldValueMap(const ZMQ_FieldValueMap * const field_value_map) const; + + u_int8_t parseOptionFieldMap(json_object * const jo) const; + u_int8_t parseOptionFieldValueMap(json_object * const jo) const; + + public: + ZMQParserInterface(const char *endpoint, const char *custom_interface_type = NULL); + ~ZMQParserInterface(); + + u_int8_t parseFlow(const char * const payload, int payload_size, u_int8_t source_id, void *data); + u_int8_t parseEvent(const char * const payload, int payload_size, u_int8_t source_id, void *data); + u_int8_t parseCounter(const char * const payload, int payload_size, u_int8_t source_id, void *data); + u_int8_t parseTemplate(const char * const payload, int payload_size, u_int8_t source_id, void *data); + u_int8_t parseOption(const char * const payload, int payload_size, u_int8_t source_id, void *data); + + virtual void setRemoteStats(ZMQ_RemoteStats *zrs); +#ifdef NTOPNG_PRO + virtual bool getCustomAppDetails(u_int32_t remapped_app_id, u_int32_t *const pen, u_int32_t *const app_field, u_int32_t *const app_id); +#endif + u_int32_t getNumDroppedPackets() { return zmq_remote_stats ? zmq_remote_stats->sflow_pkt_sample_drops : 0; }; + virtual void lua(lua_State* vm); +}; + +#endif /* _ZMQ_PARSER_INTERFACE_H_ */ + + diff --git a/include/ntop_defines.h b/include/ntop_defines.h index c4929264743e..fcbde0482788 100644 --- a/include/ntop_defines.h +++ b/include/ntop_defines.h @@ -247,6 +247,7 @@ #define CONST_INTERFACE_TYPE_PCAP "pcap" #define CONST_INTERFACE_TYPE_PCAP_DUMP "pcap dump" #define CONST_INTERFACE_TYPE_ZMQ "zmq" +#define CONST_INTERFACE_TYPE_SYSLOG "syslog" #define CONST_INTERFACE_TYPE_VLAN "Dynamic VLAN" #define CONST_INTERFACE_TYPE_FLOW "Dynamic Flow Collection" #define CONST_INTERFACE_TYPE_VIEW "view" diff --git a/include/ntop_includes.h b/include/ntop_includes.h index e37e4b860d85..3d5193e45acf 100644 --- a/include/ntop_includes.h +++ b/include/ntop_includes.h @@ -308,7 +308,10 @@ using namespace std; #endif #ifndef HAVE_NEDGE #include "ParserInterface.h" -#include "CollectorInterface.h" +#include "ZMQParserInterface.h" +#include "ZMQCollectorInterface.h" +#include "SyslogParserInterface.h" +#include "SyslogCollectorInterface.h" #include "ZCCollectorInterface.h" #include "DummyInterface.h" #include "ExportInterface.h" diff --git a/include/ntop_typedefs.h b/include/ntop_typedefs.h index 808d8eabddf5..5dc2553727dd 100644 --- a/include/ntop_typedefs.h +++ b/include/ntop_typedefs.h @@ -206,7 +206,7 @@ typedef struct zmq_flow_core { /* Extensions used only during serialization */ u_int16_t extn_len; //char extn[]; -} ZMQ_FlowCore; +} Parsed_FlowCore; /* Handle vendor-proprietary applications. Must stay with 32-bit integers as, at least sonicwall, uses @@ -218,14 +218,14 @@ typedef struct { } custom_app_t; typedef struct zmq_flow { - ZMQ_FlowCore core; + Parsed_FlowCore core; json_object *additional_fields; char *http_url, *http_site, *dns_query, *ssl_server_name, *bittorrent_hash; custom_app_t custom_app; /* Process Extensions */ -} ZMQ_Flow; +} Parsed_Flow; -/* IMPORTANT: whenever the ZMQ_FlowSerial is changed, nProbe must be updated too */ +/* IMPORTANT: whenever the Parsed_FlowSerial is changed, nProbe must be updated too */ typedef struct zmq_remote_stats { @@ -569,7 +569,8 @@ typedef enum { interface_type_NETFILTER, interface_type_DIVERT, interface_type_DUMMY, - interface_type_ZC_FLOW + interface_type_ZC_FLOW, + interface_type_SYSLOG } InterfaceType; /* Update Flow::dissectHTTP when extending the type below */ diff --git a/scripts/lua/host_details.lua b/scripts/lua/host_details.lua index dc8d57d87c4a..e060dd1917fe 100644 --- a/scripts/lua/host_details.lua +++ b/scripts/lua/host_details.lua @@ -1934,7 +1934,8 @@ drawGraphs(ifId, schema, tags, _GET["zoom"], url, selected_epoch, { {schema="host:host_unreachable_flows", label="Host Unreachable Flows"}, {schema="host:dns_sent", label="DNS Packets Sent"}, {schema="host:dns_rcvd", label="DNS Packets Rcvd"}, - {schema="host:udp_pkts", label="UDP Packets"}, + {schema="host:udp_pkts", label="UDP Packets"}, + {schema="host:tcp_stats", label="TCP Stats"}, {schema="host:1d_delta_traffic_volume", label="1 Day Traffic Delta"}, -- TODO localize {schema="host:1d_delta_flows", label="1 Day Active Flows Delta"}, -- TODO localize diff --git a/scripts/lua/modules/timeseries/schemas/ts_5min.lua b/scripts/lua/modules/timeseries/schemas/ts_5min.lua index 4c7d40024309..267a284e45d1 100644 --- a/scripts/lua/modules/timeseries/schemas/ts_5min.lua +++ b/scripts/lua/modules/timeseries/schemas/ts_5min.lua @@ -223,6 +223,15 @@ schema:addMetric("replies_error_packets") --############################################## +schema = ts_utils.newSchema("host:tcp_stats", {step = 300}) +schema:addTag("ifid") +schema:addTag("host") +schema:addMetric("retransmission_packets") +schema:addMetric("out_of_order_packets") +schema:addMetric("lost_packets") + +--############################################## + schema = ts_utils.newSchema("host:dns_rcvd", {step = 300}) schema:addTag("ifid") schema:addTag("host") diff --git a/scripts/lua/modules/ts_5min_dump_utils.lua b/scripts/lua/modules/ts_5min_dump_utils.lua index 00d9c586a8d2..bf9d97bab05d 100644 --- a/scripts/lua/modules/ts_5min_dump_utils.lua +++ b/scripts/lua/modules/ts_5min_dump_utils.lua @@ -225,6 +225,13 @@ function ts_dump.host_update_stats_rrds(when, hostname, host, ifstats, verbose) packets_sent = host["udp.packets.sent"], packets_rcvd = host["udp.packets.rcvd"]}, when, verbose) + + -- Tcp Stats + ts_utils.append("host:tcp_stats", {ifid = ifstats.id, host = hostname, + retransmission_packets = host["tcp.packets.retransmissions"], + out_of_order_packets = host["tcp.packets.out_of_order"], + lost_packets = host["tcp.packets.lost"]}, + when, verbose) -- Total number of alerts ts_utils.append("host:total_alerts", {ifid = ifstats.id, host = hostname, diff --git a/src/ArpStatsHashMatrix.cpp b/src/ArpStatsHashMatrix.cpp index 6b58bb50005e..3408a0550e94 100644 --- a/src/ArpStatsHashMatrix.cpp +++ b/src/ArpStatsHashMatrix.cpp @@ -23,42 +23,39 @@ #include "ntop_includes.h" -ArpStatsHashMatrix::ArpStatsHashMatrix(NetworkInterface *_iface, u_int _num_hashes, u_int _max_hash_size) : +ArpStatsHashMatrix::ArpStatsHashMatrix(NetworkInterface *_iface, + u_int _num_hashes, u_int _max_hash_size) : GenericHash(_iface, _num_hashes, _max_hash_size, "ArpStatsHashMatrix") { ; } /* ************************************ */ -//this get function DO NOT reverse the snd / rcv counters in case src_mac and dst_mac are reversed -ArpStatsMatrixElement* ArpStatsHashMatrix::get(const u_int8_t _src_mac[6], const u_int8_t _dst_mac[6], bool * const src2dst) { - if(_src_mac == NULL || _dst_mac == NULL) - return(NULL); - else { - u_int32_t hash = Utils::macHash((u_int8_t*)_src_mac) + Utils::macHash((u_int8_t*)_dst_mac); - hash %= num_hashes; - - if(table[hash] == NULL) { - return(NULL); - - } else { - ArpStatsMatrixElement *head; - - locks[hash]->lock(__FILE__, __LINE__); - head = (ArpStatsMatrixElement*)table[hash]; - - while(head != NULL) { - if((!head->idle()) && head->equal(_src_mac, _dst_mac, src2dst)) - - break; - else - head = (ArpStatsMatrixElement*)head->next(); - } + +ArpStatsMatrixElement* ArpStatsHashMatrix::get(const u_int8_t _src_mac[6], + const u_int32_t _src_ip, const u_int32_t _dst_ip, + bool * const src2dst) { + u_int32_t hash = (_src_ip + _dst_ip) % num_hashes; + + if(table[hash] == NULL) { + return(NULL); + } else { + ArpStatsMatrixElement *head; - locks[hash]->unlock(__FILE__, __LINE__); + locks[hash]->lock(__FILE__, __LINE__); + head = (ArpStatsMatrixElement*)table[hash]; - return(head); + while(head != NULL) { + if((!head->idle()) && head->equal(_src_mac, _src_ip, _dst_ip, src2dst)) + + break; + else + head = (ArpStatsMatrixElement*)head->next(); } - } + + locks[hash]->unlock(__FILE__, __LINE__); + + return(head); + } } /* ************************************ */ @@ -75,7 +72,6 @@ static bool print_all_arp_stats(GenericHashEntry *e, void *user_data, bool *matc print_all_arp_stats_data_t * print_all_arp_stats_data = (print_all_arp_stats_data_t*) user_data; lua_State* vm = print_all_arp_stats_data->vm; - //TODO: errors handling if(elem && vm) { lua_newtable(vm); diff --git a/src/ArpStatsMatrixElement.cpp b/src/ArpStatsMatrixElement.cpp index d8907d3e3d1a..00199db97fa5 100644 --- a/src/ArpStatsMatrixElement.cpp +++ b/src/ArpStatsMatrixElement.cpp @@ -21,33 +21,28 @@ #include "ntop_includes.h" -ArpStatsMatrixElement::ArpStatsMatrixElement(NetworkInterface *_iface, const u_int8_t _src_mac[6], const u_int8_t _dst_mac[6], bool * const src2dst): GenericHashEntry(_iface) { - memcpy(src_mac, _src_mac, 6); - memcpy(dst_mac, _dst_mac, 6); +// #define TRACE_ARP_LIFECYCLE 1 + +ArpStatsMatrixElement::ArpStatsMatrixElement(NetworkInterface *_iface, + const u_int8_t _src_mac[6], const u_int8_t _dst_mac[6], + const u_int32_t _src_ip, const u_int32_t _dst_ip): GenericHashEntry(_iface) { + memcpy(src_mac, _src_mac, 6), memcpy(dst_mac, _dst_mac, 6); + src_ip = _src_ip, dst_ip = _dst_ip; memset(&stats, 0, sizeof(stats)); - *src2dst = true; -#ifdef ARP_STATS_MATRIX_ELEMENT_DEBUG - char buf1[32], buf2[32]; - ntop->getTrace()->traceEvent(TRACE_NORMAL, "ADDED ArpMatrixElement: SourceMac %s - DestinationMac %s [num_elements: %u]", - Utils::formatMac(src_mac, buf1, sizeof(buf1)), - Utils::formatMac(dst_mac, buf2, sizeof(buf2)), - iface->getNumArpStatsMatrixElements()); +#ifdef TRACE_ARP_LIFECYCLE + print((char*)"Create "); #endif } /* *************************************** */ -ArpStatsMatrixElement::~ArpStatsMatrixElement(){ -#ifdef ARP_STATS_MATRIX_ELEMENT_DEBUG - char buf1[32], buf2[32]; - ntop->getTrace()->traceEvent(TRACE_NORMAL, "DELETED ArpMatrixElement: SourceMac %s - DestinationMac %s [num_elements: %u]", - Utils::formatMac(src_mac, buf1, sizeof(buf1)), - Utils::formatMac(dst_mac, buf2, sizeof(buf2)), - iface->getNumArpStatsMatrixElements()); +ArpStatsMatrixElement::~ArpStatsMatrixElement() { +#ifdef TRACE_ARP_LIFECYCLE + print((char*)"Delete "); #endif - } + /* *************************************** */ bool ArpStatsMatrixElement::idle() { @@ -63,16 +58,36 @@ bool ArpStatsMatrixElement::idle() { /* *************************************** */ -bool ArpStatsMatrixElement::equal(const u_int8_t _src_mac[6], const u_int8_t _dst_mac[6], bool * const src2dst) const { - if(!_src_mac || !_dst_mac) - return false; - - if(memcmp(src_mac, _src_mac, 6) == 0 && memcmp(dst_mac, _dst_mac, 6) == 0) { +bool ArpStatsMatrixElement::equal(const u_int8_t _src_mac[6], + const u_int32_t _src_ip, const u_int32_t _dst_ip, + bool * const src2dst) const { + + if((src_ip == _src_ip) && (dst_ip == _dst_ip)) { + if(memcmp(src_mac, _src_mac, 6) != 0) { + /* This is a new Mac */ + memcpy((void*)src_mac, _src_mac, 6); /* Overwrite Mac (e.g. DHCP reassignment) */ + memset((void*)&stats, 0, sizeof(stats)); /* Reset all stats */ + } + *src2dst = true; return true; - } else if(memcmp(src_mac, _dst_mac, 6) == 0 && memcmp(dst_mac, _src_mac, 6) == 0) { - *src2dst = false; - return true; + } else { + u_int8_t empty_mac[6] = { 0 }; + + if((src_ip == _dst_ip) && (dst_ip == _src_ip)) { + if(memcmp(dst_mac, _src_mac, 6) == 0) + ; /* Same mac: nothing to do */ + else if (memcmp(dst_mac, empty_mac, 6) == 0) + memcpy((void*)dst_mac, _src_mac, 6); /* Mac was never set */ + else { + /* This is a new Mac */ + memcpy((void*)dst_mac, _src_mac, 6); /* Overwrite Mac (e.g. DHCP reassignment) */ + memset((void*)&stats, 0, sizeof(stats)); /* Reset all stats */ + } + + *src2dst = false; + return true; + } } return false; @@ -81,40 +96,42 @@ bool ArpStatsMatrixElement::equal(const u_int8_t _src_mac[6], const u_int8_t _ds /* *************************************** */ u_int32_t ArpStatsMatrixElement::key() { - return Utils::macHash(src_mac) + Utils::macHash(dst_mac); + return(src_ip + dst_ip); } /* *************************************** */ -void ArpStatsMatrixElement::print() const { - char buf1[32], buf2[32]; - ntop->getTrace()->traceEvent(TRACE_NORMAL, "[SourceMac: %s][DestinationMac: %s]", +void ArpStatsMatrixElement::print(char *msg) const { + char buf1[32], buf1ip[32], buf2[32], buf2ip[32]; + ntop->getTrace()->traceEvent(TRACE_NORMAL, "%s[Source: %s/%s][Dest: %s/%s]", + msg ? msg : "", Utils::formatMac(src_mac, buf1, sizeof(buf1)), - Utils::formatMac(dst_mac, buf2, sizeof(buf2))); + Utils::intoaV4(src_ip, buf1ip, sizeof(buf1ip)), + Utils::formatMac(dst_mac, buf2, sizeof(buf2)), + Utils::intoaV4(dst_ip, buf2ip, sizeof(buf2ip)) + ); } /* *************************************** */ void ArpStatsMatrixElement::lua(lua_State* vm) { - char buf[32]; + char buf[32], buf1[32], key[64]; - lua_newtable(vm); /* Outer table key, source mac */ - lua_newtable(vm); /* Innter table key, destination mac */ + lua_newtable(vm); + lua_push_str_table_entry(vm, "src_mac", Utils::formatMac(src_mac, buf, sizeof(buf))); + lua_push_str_table_entry(vm, "dst_mac", Utils::formatMac(dst_mac, buf, sizeof(buf))); + lua_push_uint64_table_entry(vm, "src2dst.requests", stats.src2dst.requests); lua_push_uint64_table_entry(vm, "src2dst.replies", stats.src2dst.replies); lua_push_uint64_table_entry(vm, "dst2src.requests", stats.dst2src.requests); lua_push_uint64_table_entry(vm, "dst2src.replies", stats.dst2src.replies); - /* Destination Mac as the key of the inner table */ - Utils::formatMac(dst_mac, buf, sizeof(buf)); - lua_pushstring(vm, buf); - lua_insert(vm, -2); - lua_settable(vm, -3); + snprintf(key, sizeof(key), "%s-%s", + Utils::intoaV4(src_ip, buf, sizeof(buf)), + Utils::intoaV4(dst_ip, buf1, sizeof(buf1))); - /* Source Mac as the key of the outer table */ - Utils::formatMac(src_mac, buf, sizeof(buf)); - lua_pushstring(vm, buf); + lua_pushstring(vm, key); lua_insert(vm, -2); lua_settable(vm, -3); } diff --git a/src/DummyInterface.cpp b/src/DummyInterface.cpp index 4634d784a7ed..25b806bce9bf 100644 --- a/src/DummyInterface.cpp +++ b/src/DummyInterface.cpp @@ -25,7 +25,7 @@ /* **************************************************** */ -DummyInterface::DummyInterface() : ParserInterface("dummy") { +DummyInterface::DummyInterface() : ZMQParserInterface("dummy") { ntop->getTrace()->traceEvent(TRACE_NORMAL, "Initialized dummy interface"); } diff --git a/src/Flow.cpp b/src/Flow.cpp index 4ee1da99be0f..3e3f579561fe 100644 --- a/src/Flow.cpp +++ b/src/Flow.cpp @@ -46,7 +46,7 @@ Flow::Flow(NetworkInterface *_iface, srv2cli_last_goodput_bytes = cli2srv_last_goodput_bytes = 0, good_ssl_hs = true, flow_alerted = flow_dropped_counts_increased = false, vrfId = 0; - l7_protocol_guessed = detection_completed = false; + detection_completed = false; ndpiDetectedProtocol = ndpiUnknownProtocol; doNotExpireBefore = iface->getTimeLastPktRcvd() + DONT_NOT_EXPIRE_BEFORE_SEC; @@ -553,65 +553,49 @@ void Flow::processDetectedProtocol() { /* *************************************** */ -void Flow::guessProtocol() { - if(detection_completed) - return; /* Nothing to do */ - - /* This code should no longer be necessary as the nDPI API changed */ - if((protocol == IPPROTO_TCP) || (protocol == IPPROTO_UDP)) { - if(cli_host && srv_host) { - /* We can guess the protocol */ - IpAddress *cli_ip = cli_host->get_ip(), *srv_ip = srv_host->get_ip(); - ndpi_protocol guessed_proto = ndpi_guess_undetected_protocol(iface->get_ndpi_struct(), NULL, protocol, - ntohl(cli_ip ? cli_ip->get_ipv4() : 0), - ntohs(cli_port), - ntohl(srv_ip ? srv_ip->get_ipv4() : 0), - ntohs(srv_port)); - ndpiDetectedProtocol.master_protocol = guessed_proto.master_protocol; - ndpiDetectedProtocol.app_protocol = guessed_proto.app_protocol; - - /* NOTE: only overwrite the category if it was not set. - * This prevents overwriting already determined category (e.g. by IP or Host) - */ - if(ndpiDetectedProtocol.category == NDPI_PROTOCOL_CATEGORY_UNSPECIFIED) - ndpiDetectedProtocol.category = guessed_proto.category; - } - - l7_protocol_guessed = true; +void Flow::setDetectedProtocol(ndpi_protocol proto_id, bool forceDetection) { + ndpi_flow_struct* ndpif; + + /* Let the client SSL certificate win over the server SSL certificate + this addresses detection for youtube, e.g., when the client + requests s.youtube.com but the server responds with google.com */ + if(!forceDetection + && proto_id.master_protocol == NDPI_PROTOCOL_SSL + && get_packets() < NDPI_MIN_NUM_PACKETS + && (ndpif = get_ndpi_flow()) + && ndpif->protos.stun_ssl.ssl.client_certificate[0] == '\0' + && ndpif->protos.stun_ssl.ssl.server_certificate[0] == '\0') { + ndpif->detected_protocol_stack[0] = NDPI_PROTOCOL_UNKNOWN; + return; } - detection_completed = true; /* We give up */ -} + if(proto_id.app_protocol != NDPI_PROTOCOL_UNKNOWN + || forceDetection + || get_packets() >= NDPI_MIN_NUM_PACKETS + || !iface->is_ndpi_enabled() + || iface->isSampledTraffic()) { + ndpiDetectedProtocol.master_protocol = proto_id.master_protocol; + ndpiDetectedProtocol.app_protocol = proto_id.app_protocol; -/* *************************************** */ - -void Flow::setDetectedProtocol(ndpi_protocol proto_id, bool forceDetection) { - ndpiDetectedProtocol.category = proto_id.category; - - if(proto_id.app_protocol != NDPI_PROTOCOL_UNKNOWN) { - ndpiDetectedProtocol = proto_id; - - /* Let the client SSL certificate win over the server SSL certificate - this addresses detection for youtube, e.g., when the client - requests s.youtube.com but the server responds with google.com */ - if((proto_id.master_protocol == NDPI_PROTOCOL_SSL) - && (get_packets() < NDPI_MIN_NUM_PACKETS) - && (ndpiFlow) - && (ndpiFlow->protos.stun_ssl.ssl.client_certificate[0] == '\0') - && (ndpiFlow->protos.stun_ssl.ssl.server_certificate[0] == '\0')) { - get_ndpi_flow()->detected_protocol_stack[0] = NDPI_PROTOCOL_UNKNOWN; - return; - } + /* NOTE: only overwrite the category if it was not set. + * This prevents overwriting already determined category (e.g. by IP or Host) + */ + if(ndpiDetectedProtocol.category == NDPI_PROTOCOL_CATEGORY_UNSPECIFIED) + ndpiDetectedProtocol.category = proto_id.category; processDetectedProtocol(); detection_completed = true; - } else if(forceDetection - || (get_packets() >= NDPI_MIN_NUM_PACKETS) - || (!iface->is_ndpi_enabled()) - || iface->isSampledTraffic() - ) - guessProtocol(); - + +#ifdef BLACKLISTED_FLOWS_DEBUG + if(ndpiDetectedProtocol.category == CUSTOM_CATEGORY_MALWARE) { + char buf[512]; + print(buf, sizeof(buf)); + snprintf(&buf[strlen(buf)], sizeof(buf) - strlen(buf), "Malware category detected. [cli_blacklisted: %u][srv_blacklisted: %u][category: %s]", cli_host->isBlacklisted(), srv_host->isBlacklisted(), get_protocol_category_name()); + ntop->getTrace()->traceEvent(TRACE_NORMAL, "%s", buf); + } +#endif + } + if(detection_completed) { #ifdef HAVE_NEDGE updateFlowShapers(true); @@ -764,7 +748,7 @@ char* Flow::printTCPflags(u_int8_t flags, char * const buf, u_int buf_len) const } /* *************************************** */ -char* Flow::print(char *buf, u_int buf_len) { +char* Flow::print(char *buf, u_int buf_len) const { char buf1[32], buf2[32], buf3[32], buf4[32], pbuf[32], tcp_buf[64]; buf[0] = '\0'; @@ -850,9 +834,6 @@ char* Flow::print(char *buf, u_int buf_len) { #endif ); - if(getFlowStatus() == status_dns_invalid_query && protos.dns.last_query) - snprintf(&buf[strlen(buf)], buf_len - strlen(buf), "[query: %s]", protos.dns.last_query); - return(buf); } @@ -2320,8 +2301,10 @@ void Flow::housekeep(time_t t) { if(!isDetectionCompleted() && t - get_last_seen() > 5 /* sec */ && iface->get_ndpi_struct() - && get_ndpi_flow()) - setDetectedProtocol(ndpi_detection_giveup(iface->get_ndpi_struct(), get_ndpi_flow(), 1), true); + && get_ndpi_flow()) { + ndpi_protocol givenup_protocol = ndpi_detection_giveup(iface->get_ndpi_struct(), get_ndpi_flow(), 1); + setDetectedProtocol(givenup_protocol, true); + } } /* *************************************** */ @@ -2361,6 +2344,26 @@ void Flow::dumpPacketStats(lua_State* vm, bool cli2srv_direction) { /* *************************************** */ +bool Flow::isBlacklistedFlow() const { + bool res = (cli_host && srv_host + && (cli_host->isBlacklisted() + || srv_host->isBlacklisted() + || (get_protocol_category() == CUSTOM_CATEGORY_MALWARE))); + +#ifdef BLACKLISTED_FLOWS_DEBUG + if(res) { + char buf[512]; + print(buf, sizeof(buf)); + snprintf(&buf[strlen(buf)], sizeof(buf) - strlen(buf), "[cli_blacklisted: %u][srv_blacklisted: %u][category: %s]", cli_host->isBlacklisted(), srv_host->isBlacklisted(), get_protocol_category_name()); + ntop->getTrace()->traceEvent(TRACE_NORMAL, "%s", buf); + } +#endif + + return res; +}; + +/* *************************************** */ + bool Flow::isSSLProto() { u_int16_t lower = ndpi_get_lower_proto(ndpiDetectedProtocol); diff --git a/src/Host.cpp b/src/Host.cpp index 9e599808899b..061996a98016 100644 --- a/src/Host.cpp +++ b/src/Host.cpp @@ -557,7 +557,7 @@ void Host::lua(lua_State* vm, AddressTree *ptree, lua_push_str_table_entry(vm, "asname", asname ? asname : (char*)""); lua_push_str_table_entry(vm, "os", get_os(buf, sizeof(buf))); - stats->lua(vm, mask_host, host_details, verbose); + stats->lua(vm, mask_host, Utils::bool2DetailsLevel(verbose,host_details)); lua_push_uint64_table_entry(vm, "anomalous_flows.as_server", getTotalNumAnomalousIncomingFlows()); lua_push_uint64_table_entry(vm, "anomalous_flows.as_client", getTotalNumAnomalousOutgoingFlows()); diff --git a/src/HostStats.cpp b/src/HostStats.cpp index 285a19614ae0..72f8a9c52929 100644 --- a/src/HostStats.cpp +++ b/src/HostStats.cpp @@ -84,10 +84,11 @@ void HostStats::getJSONObject(json_object *my_object, DetailsLevel details_level /* *************************************** */ -void HostStats::lua(lua_State* vm, bool mask_host, bool host_details, bool verbose, bool tsLua) { - lua_push_uint64_table_entry(vm, "bytes.ndpi.unknown", getnDPIStats() ? getnDPIStats()->getProtoBytes(NDPI_PROTOCOL_UNKNOWN) : 0); +void HostStats::lua(lua_State* vm, bool mask_host, DetailsLevel details_level, bool tsLua) { + if(details_level >= details_high) + lua_push_uint64_table_entry(vm, "bytes.ndpi.unknown", getnDPIStats() ? getnDPIStats()->getProtoBytes(NDPI_PROTOCOL_UNKNOWN) : 0); - if(verbose) { + if(details_level >= details_max) { #ifdef NTOPNG_PRO if(custom_app_stats) custom_app_stats->lua(vm); #endif @@ -97,15 +98,21 @@ void HostStats::lua(lua_State* vm, bool mask_host, bool host_details, bool verbo } /* TCP stats */ - if(host_details) { + if(details_level >= details_higher) { lua_push_bool_table_entry(vm, "tcp.packets.seq_problems", (tcpPacketStats.pktRetr || tcpPacketStats.pktOOO || tcpPacketStats.pktLost || tcpPacketStats.pktKeepAlive) ? true : false); + } + + if(details_level != details_high){ lua_push_uint64_table_entry(vm, "tcp.packets.retransmissions", tcpPacketStats.pktRetr); lua_push_uint64_table_entry(vm, "tcp.packets.out_of_order", tcpPacketStats.pktOOO); lua_push_uint64_table_entry(vm, "tcp.packets.lost", tcpPacketStats.pktLost); + } + + if(details_level >= details_higher) { lua_push_uint64_table_entry(vm, "tcp.packets.keep_alive", tcpPacketStats.pktKeepAlive); /* Bytes anomalies */ @@ -117,7 +124,11 @@ void HostStats::lua(lua_State* vm, bool mask_host, bool host_details, bool verbo lua_push_uint64_table_entry(vm, "icmp.bytes.rcvd.anomaly_index", icmp_sent.getBytesAnomaly()); lua_push_uint64_table_entry(vm, "other_ip.bytes.sent.anomaly_index", other_ip_sent.getBytesAnomaly()); lua_push_uint64_table_entry(vm, "other_ip.bytes.rcvd.anomaly_index", other_ip_sent.getBytesAnomaly()); - } else { + + lua_push_uint64_table_entry(vm, "total_activity_time", total_activity_time); + lua_push_uint64_table_entry(vm, "flows.as_client", getTotalNumFlowsAsClient()); + lua_push_uint64_table_entry(vm, "flows.as_server", getTotalNumFlowsAsServer()); + } else if(details_level >= details_high) { /* Limit tcp information to anomalies when host_details aren't required */ if(tcpPacketStats.pktRetr) lua_push_uint64_table_entry(vm, "tcp.packets.retransmissions", tcpPacketStats.pktRetr); @@ -129,14 +140,10 @@ void HostStats::lua(lua_State* vm, bool mask_host, bool host_details, bool verbo lua_push_uint64_table_entry(vm, "tcp.packets.keep_alive", tcpPacketStats.pktKeepAlive); } - if(host_details) { - lua_push_uint64_table_entry(vm, "total_activity_time", total_activity_time); - lua_push_uint64_table_entry(vm, "flows.as_client", getTotalNumFlowsAsClient()); - lua_push_uint64_table_entry(vm, "flows.as_server", getTotalNumFlowsAsServer()); + if(details_level >= details_high){ + ((GenericTrafficElement*)this)->lua(vm, details_level >= details_higher); + ((TimeseriesStats*)this)->luaStats(vm, iface, details_level >= details_higher, details_level >= details_max, tsLua); } - - ((GenericTrafficElement*)this)->lua(vm, host_details); - ((TimeseriesStats*)this)->luaStats(vm, iface, host_details, verbose, tsLua); } /* *************************************** */ diff --git a/src/LocalHostStats.cpp b/src/LocalHostStats.cpp index bc26c2860dd9..47b27beae02c 100644 --- a/src/LocalHostStats.cpp +++ b/src/LocalHostStats.cpp @@ -115,8 +115,8 @@ void LocalHostStats::getJSONObject(json_object *my_object, DetailsLevel details_ /* *************************************** */ -void LocalHostStats::lua(lua_State* vm, bool mask_host, bool host_details, bool verbose, bool tsLua) { - HostStats::lua(vm, mask_host, host_details, verbose, tsLua); +void LocalHostStats::lua(lua_State* vm, bool mask_host, DetailsLevel details_level, bool tsLua) { + HostStats::lua(vm, mask_host, details_level, tsLua); if((!mask_host) && top_sites && ntop->getPrefs()->are_top_talkers_enabled()) { char *cur_sites = top_sites->json(); @@ -125,14 +125,10 @@ void LocalHostStats::lua(lua_State* vm, bool mask_host, bool host_details, bool if(cur_sites) free(cur_sites); } - if(host_details) { - if(icmp) - icmp->lua(host->get_ip()->isIPv4(), vm); - } - - if(verbose) { - if(dns) luaDNS(vm,true); - if(http) http->lua(vm); + if(details_level >= details_high) { + if(icmp) icmp->lua(host->get_ip()->isIPv4(), vm); + if(dns) luaDNS(vm, true); + if(http) http->lua(vm); } } diff --git a/src/NetworkInterface.cpp b/src/NetworkInterface.cpp index 5d31cec09dad..95aff3fdea65 100644 --- a/src/NetworkInterface.cpp +++ b/src/NetworkInterface.cpp @@ -1035,7 +1035,7 @@ NetworkInterface* NetworkInterface::getSubInterface(u_int32_t criteria, bool par /* **************************************************** */ -void NetworkInterface::processFlow(ZMQ_Flow *zflow) { +void NetworkInterface::processFlow(Parsed_Flow *zflow) { bool src2dst_direction, new_flow; Flow *flow; ndpi_protocol p; @@ -1652,24 +1652,20 @@ bool NetworkInterface::processPacket(u_int32_t bridge_iface_idx, flow->updateInterfaceLocalStats(src2dst_direction, 1, rawsize); if(!flow->isDetectionCompleted()) { - if(isSampledTraffic()) - flow->guessProtocol(); - else { - if(!is_fragment) { - struct ndpi_flow_struct *ndpi_flow = flow->get_ndpi_flow(); - struct ndpi_id_struct *cli = (struct ndpi_id_struct*)flow->get_cli_id(); - struct ndpi_id_struct *srv = (struct ndpi_id_struct*)flow->get_srv_id(); + if(!is_fragment) { + struct ndpi_flow_struct *ndpi_flow = flow->get_ndpi_flow(); + struct ndpi_id_struct *cli = (struct ndpi_id_struct*)flow->get_cli_id(); + struct ndpi_id_struct *srv = (struct ndpi_id_struct*)flow->get_srv_id(); - if(flow->get_packets() >= NDPI_MIN_NUM_PACKETS) { - flow->setDetectedProtocol(ndpi_detection_giveup(ndpi_struct, ndpi_flow, 1), false); - } else - flow->setDetectedProtocol(ndpi_detection_process_packet(ndpi_struct, ndpi_flow, - ip, ipsize, (u_int32_t)packet_time, - cli, srv), false); - } else { - // FIX - only handle unfragmented packets - // ntop->getTrace()->traceEvent(TRACE_WARNING, "IP fragments are not handled yet!"); - } + if(flow->get_packets() >= NDPI_MIN_NUM_PACKETS) { + flow->setDetectedProtocol(ndpi_detection_giveup(ndpi_struct, ndpi_flow, 1), false); + } else + flow->setDetectedProtocol(ndpi_detection_process_packet(ndpi_struct, ndpi_flow, + ip, ipsize, (u_int32_t)packet_time, + cli, srv), false); + } else { + // FIX - only handle unfragmented packets + // ntop->getTrace()->traceEvent(TRACE_WARNING, "IP fragments are not handled yet!"); } } @@ -2577,7 +2573,9 @@ bool NetworkInterface::dissectPacket(u_int32_t bridge_iface_idx, } } - e = getArpHashMatrixElement(srcMac->get_mac(), dstMac->get_mac(), &src2dst_element); + e = getArpHashMatrixElement(srcMac->get_mac(), dstMac->get_mac(), + src, dst, + &src2dst_element); if(arp_opcode == 0x1 /* ARP request */) { arp_requests++; @@ -5576,19 +5574,20 @@ Mac* NetworkInterface::getMac(u_int8_t _mac[6], bool createIfNotPresent) { /* **************************************************** */ -ArpStatsMatrixElement* NetworkInterface::getArpHashMatrixElement(const u_int8_t _src_mac[6], - const u_int8_t _dst_mac[6], - bool * const src2dst){ +ArpStatsMatrixElement* NetworkInterface::getArpHashMatrixElement(const u_int8_t _src_mac[6], const u_int8_t _dst_mac[6], + const u_int32_t _src_ip, const u_int32_t _dst_ip, + bool * const src2dst) { ArpStatsMatrixElement *ret = NULL; - if(_src_mac == NULL || _dst_mac == NULL || arp_hash_matrix == NULL) + if(arp_hash_matrix == NULL) return NULL; - ret = arp_hash_matrix->get(_src_mac, _dst_mac, src2dst); + ret = arp_hash_matrix->get(_src_mac, _src_ip, _dst_ip, src2dst); if(ret == NULL) { try{ - if((ret = new ArpStatsMatrixElement(this, _src_mac, _dst_mac, src2dst)) != NULL) + if((ret = new ArpStatsMatrixElement(this, _src_mac, _dst_mac, _src_ip, _dst_ip)) != NULL) + if(!arp_hash_matrix->add(ret)){ delete ret; ret = NULL; @@ -5597,8 +5596,8 @@ ArpStatsMatrixElement* NetworkInterface::getArpHashMatrixElement(const u_int8_t static bool oom_warning_sent = false; if(!oom_warning_sent) { - ntop->getTrace()->traceEvent(TRACE_WARNING, "Not enough memory"); - oom_warning_sent = true; + ntop->getTrace()->traceEvent(TRACE_WARNING, "Not enough memory"); + oom_warning_sent = true; } return(NULL); } diff --git a/src/ParserInterface.cpp b/src/ParserInterface.cpp index 29f80fa82660..306b892fabc9 100755 --- a/src/ParserInterface.cpp +++ b/src/ParserInterface.cpp @@ -25,1002 +25,14 @@ /* **************************************************** */ -/* IMPORTANT: keep it in sync with flow_fields_description part of flow_utils.lua */ ParserInterface::ParserInterface(const char *endpoint, const char *custom_interface_type) : NetworkInterface(endpoint, custom_interface_type) { - zmq_initial_bytes = 0, zmq_initial_pkts = 0; - zmq_remote_stats = zmq_remote_stats_shadow = NULL; - zmq_remote_initial_exported_flows = 0; - once = false; -#ifdef NTOPNG_PRO - custom_app_maps = NULL; -#endif - - /* Populate defaults for @NTOPNG@ nProbe templates. No need to populate - all the fields as nProbe will sent them periodically. - This minimum set is required for backward compatibility. */ - addMapping("IN_SRC_MAC", IN_SRC_MAC); - addMapping("OUT_SRC_MAC", OUT_SRC_MAC); - addMapping("IN_DST_MAC", IN_DST_MAC); - addMapping("OUT_DST_MAC", OUT_DST_MAC); - addMapping("SRC_VLAN", SRC_VLAN); - addMapping("DST_VLAN", DST_VLAN); - addMapping("DOT1Q_SRC_VLAN", DOT1Q_SRC_VLAN); - addMapping("DOT1Q_DST_VLAN", DOT1Q_DST_VLAN); - addMapping("INPUT_SNMP", INPUT_SNMP); - addMapping("OUTPUT_SNMP", OUTPUT_SNMP); - addMapping("IPV4_SRC_ADDR", IPV4_SRC_ADDR); - addMapping("IPV4_DST_ADDR", IPV4_DST_ADDR); - addMapping("L4_SRC_PORT", L4_SRC_PORT); - addMapping("L4_DST_PORT", L4_DST_PORT); - addMapping("IPV6_SRC_ADDR", IPV6_SRC_ADDR); - addMapping("IPV6_DST_ADDR", IPV6_DST_ADDR); - addMapping("IP_PROTOCOL_VERSION", IP_PROTOCOL_VERSION); - addMapping("PROTOCOL", PROTOCOL); - addMapping("L7_PROTO", L7_PROTO, NTOP_PEN); - addMapping("IN_BYTES", IN_BYTES); - addMapping("IN_PKTS", IN_PKTS); - addMapping("OUT_BYTES", OUT_BYTES); - addMapping("OUT_PKTS", OUT_PKTS); - addMapping("FIRST_SWITCHED", FIRST_SWITCHED); - addMapping("LAST_SWITCHED", LAST_SWITCHED); - addMapping("EXPORTER_IPV4_ADDRESS", EXPORTER_IPV4_ADDRESS); - addMapping("EXPORTER_IPV6_ADDRESS", EXPORTER_IPV6_ADDRESS); - addMapping("NPROBE_IPV4_ADDRESS", NPROBE_IPV4_ADDRESS, NTOP_PEN); - addMapping("TCP_FLAGS", TCP_FLAGS); - addMapping("INITIATOR_PKTS", INITIATOR_PKTS); - addMapping("INITIATOR_OCTETS", INITIATOR_OCTETS); - addMapping("RESPONDER_PKTS", RESPONDER_PKTS); - addMapping("RESPONDER_OCTETS", RESPONDER_OCTETS); - addMapping("SAMPLING_INTERVAL", SAMPLING_INTERVAL); - addMapping("DIRECTION", DIRECTION); - addMapping("POST_NAT_SRC_IPV4_ADDR", POST_NAT_SRC_IPV4_ADDR); - addMapping("POST_NAT_DST_IPV4_ADDR", POST_NAT_DST_IPV4_ADDR); - addMapping("POST_NAPT_SRC_TRANSPORT_PORT", POST_NAPT_SRC_TRANSPORT_PORT); - addMapping("POST_NAPT_DST_TRANSPORT_PORT", POST_NAPT_DST_TRANSPORT_PORT); - addMapping("INGRESS_VRFID", INGRESS_VRFID); - addMapping("IPV4_SRC_MASK", IPV4_SRC_MASK); - addMapping("IPV4_DST_MASK", IPV4_DST_MASK); - addMapping("IPV4_NEXT_HOP", IPV4_NEXT_HOP); - addMapping("OOORDER_IN_PKTS", OOORDER_IN_PKTS, NTOP_PEN); - addMapping("OOORDER_OUT_PKTS", OOORDER_OUT_PKTS, NTOP_PEN); - addMapping("RETRANSMITTED_IN_PKTS", RETRANSMITTED_IN_PKTS, NTOP_PEN); - addMapping("RETRANSMITTED_OUT_PKTS", RETRANSMITTED_OUT_PKTS, NTOP_PEN); - addMapping("DNS_QUERY", DNS_QUERY, NTOP_PEN); - addMapping("HTTP_URL", HTTP_URL, NTOP_PEN); - addMapping("HTTP_SITE", HTTP_SITE, NTOP_PEN); - addMapping("SSL_SERVER_NAME", SSL_SERVER_NAME, NTOP_PEN); - addMapping("BITTORRENT_HASH", BITTORRENT_HASH, NTOP_PEN); } /* **************************************************** */ ParserInterface::~ParserInterface() { - if(zmq_remote_stats) free(zmq_remote_stats); - if(zmq_remote_stats_shadow) free(zmq_remote_stats_shadow); -#ifdef NTOPNG_PRO - if(custom_app_maps) delete(custom_app_maps); -#endif -} - -/* **************************************************** */ - -void ParserInterface::addMapping(const char *sym, u_int32_t num, u_int32_t pen) { - string label(sym); - labels_map_t::iterator it; - - if((it = labels_map.find(label)) == labels_map.end()) - labels_map.insert(make_pair(label, make_pair(pen, num))); - else - it->second.first = pen, it->second.second = num; -} - -/* **************************************************** */ - -bool ParserInterface::getKeyId(char *sym, u_int32_t * const pen, u_int32_t * const field) const { - u_int32_t cur_pen, cur_field; - string label(sym); - labels_map_t::const_iterator it; - - *pen = UNKNOWN_PEN, *field = UNKNOWN_FLOW_ELEMENT; - - if(sscanf(sym, "%u.%u", &cur_pen, &cur_field) == 2) - *pen = cur_pen, *field = cur_field; - else if(sscanf(sym, "%u", &cur_field) == 1) - *pen = 0, *field = cur_field; - else if((it = labels_map.find(label)) != labels_map.end()) - *pen = it->second.first, *field = it->second.second; - else - return false; - - return true; -} - -/* **************************************************** */ - -u_int8_t ParserInterface::parseEvent(const char * const payload, int payload_size, - u_int8_t source_id, void *data) { - json_object *o; - enum json_tokener_error jerr = json_tokener_success; - NetworkInterface * iface = (NetworkInterface*)data; - ZMQ_RemoteStats *zrs = NULL; - memset((void*)&zrs, 0, sizeof(zrs)); - - // payload[payload_size] = '\0'; - - // ntop->getTrace()->traceEvent(TRACE_NORMAL, "%s", payload); - o = json_tokener_parse_verbose(payload, &jerr); - - if(o && (zrs = (ZMQ_RemoteStats*)calloc(1, sizeof(ZMQ_RemoteStats)))) { - json_object *w, *z; - - if(json_object_object_get_ex(o, "time", &w)) zrs->remote_time = (u_int32_t)json_object_get_int64(w); - if(json_object_object_get_ex(o, "bytes", &w)) zrs->remote_bytes = (u_int64_t)json_object_get_int64(w); - if(json_object_object_get_ex(o, "packets", &w)) zrs->remote_pkts = (u_int64_t)json_object_get_int64(w); - - if(json_object_object_get_ex(o, "iface", &w)) { - if(json_object_object_get_ex(w, "name", &z)) - snprintf(zrs->remote_ifname, sizeof(zrs->remote_ifname), "%s", json_object_get_string(z)); - if(json_object_object_get_ex(w, "speed", &z)) - zrs->remote_ifspeed = (u_int32_t)json_object_get_int64(z); - if(json_object_object_get_ex(w, "ip", &z)) - snprintf(zrs->remote_ifaddress, sizeof(zrs->remote_ifaddress), "%s", json_object_get_string(z)); - } - - if(json_object_object_get_ex(o, "probe", &w)) { - if(json_object_object_get_ex(w, "public_ip", &z)) - snprintf(zrs->remote_probe_public_address, sizeof(zrs->remote_probe_public_address), "%s", json_object_get_string(z)); - if(json_object_object_get_ex(w, "ip", &z)) - snprintf(zrs->remote_probe_address, sizeof(zrs->remote_probe_address), "%s", json_object_get_string(z)); - } - - if(json_object_object_get_ex(o, "avg", &w)) { - if(json_object_object_get_ex(w, "bps", &z)) - zrs->avg_bps = (u_int32_t)json_object_get_int64(z); - if(json_object_object_get_ex(w, "pps", &z)) - zrs->avg_pps = (u_int32_t)json_object_get_int64(z); - } - - if(json_object_object_get_ex(o, "timeout", &w)) { - if(json_object_object_get_ex(w, "lifetime", &z)) - zrs->remote_lifetime_timeout = (u_int32_t)json_object_get_int64(z); - if(json_object_object_get_ex(w, "idle", &z)) - zrs->remote_idle_timeout = (u_int32_t)json_object_get_int64(z); - } - - if(json_object_object_get_ex(o, "drops", &w)) { - if(json_object_object_get_ex(w, "export_queue_full", &z)) - zrs->export_queue_full = (u_int32_t)json_object_get_int64(z); - - if(json_object_object_get_ex(w, "too_many_flows", &z)) - zrs->too_many_flows = (u_int32_t)json_object_get_int64(z); - - if(json_object_object_get_ex(w, "elk_flow_drops", &z)) - zrs->elk_flow_drops = (u_int32_t)json_object_get_int64(z); - - if(json_object_object_get_ex(w, "sflow_pkt_sample_drops", &z)) - zrs->sflow_pkt_sample_drops = (u_int32_t)json_object_get_int64(z); - - if(json_object_object_get_ex(w, "flow_collection_drops", &z)) - zrs->flow_collection_drops = (u_int32_t)json_object_get_int64(z); - } - - if(json_object_object_get_ex(o, "zmq", &w)) { - if(json_object_object_get_ex(w, "num_flow_exports", &z)) - zrs->num_flow_exports = (u_int64_t)json_object_get_int64(z); - - if(json_object_object_get_ex(w, "num_exporters", &z)) - zrs->num_exporters = (u_int8_t)json_object_get_int(z); - } - -#ifdef ZMQ_EVENT_DEBUG - ntop->getTrace()->traceEvent(TRACE_NORMAL, "Event parsed " - "[iface: {name: %s, speed: %u, ip:%s}]" - "[probe: {public_ip: %s, ip: %s}]" - "[avg: {bps: %u, pps: %u}]" - "[remote: {time: %u, bytes: %u, packets: %u, idle_timeout: %u, lifetime_timeout:%u}]" - "[zmq: {num_exporters: %u, num_flow_exports: %u}]", - zrs->remote_ifname, zrs->remote_ifspeed, zrs->remote_ifaddress, - zrs->remote_probe_public_address, zrs->remote_probe_address, - zrs->avg_bps, zrs->avg_pps, - zrs->remote_time, (u_int32_t)zrs->remote_bytes, (u_int32_t)zrs->remote_pkts, - zrs->remote_idle_timeout, zrs->remote_lifetime_timeout, - zrs->num_exporters, zrs->num_flow_exports); -#endif - - /* ntop->getTrace()->traceEvent(TRACE_WARNING, "%u/%u", avg_bps, avg_pps); */ - - /* Process Flow */ - static_cast(iface)->setRemoteStats(zrs); - if(flowHashing) { - FlowHashing *current, *tmp; - - HASH_ITER(hh, flowHashing, current, tmp) { - ZMQ_RemoteStats *zrscopy = (ZMQ_RemoteStats*)malloc(sizeof(ZMQ_RemoteStats)); - - if(zrscopy) - memcpy(zrscopy, zrs, sizeof(ZMQ_RemoteStats)); - - static_cast(current->iface)->setRemoteStats(zrscopy); - } - } - - /* Dispose memory */ - json_object_put(o); - } else { - // if o != NULL - if(!once) { - ntop->getTrace()->traceEvent(TRACE_WARNING, - "Invalid message received: " - "your nProbe sender is outdated, data encrypted, invalid JSON, or oom?"); - ntop->getTrace()->traceEvent(TRACE_WARNING, "JSON Parse error [%s] payload size: %u payload: %s", - json_tokener_error_desc(jerr), - payload_size, - payload); - } - once = true; - if(o) json_object_put(o); - return -1; - } - - return 0; -} - -/* **************************************************** */ - -bool ParserInterface::parsePENZeroField(ZMQ_Flow * const flow, u_int32_t field, const char * const value) const { - IpAddress ip_aux; /* used to check empty IPs */ - - switch(field) { - case IN_SRC_MAC: - case OUT_SRC_MAC: - /* Format 00:00:00:00:00:00 */ - Utils::parseMac(flow->core.src_mac, value); - break; - case IN_DST_MAC: - case OUT_DST_MAC: - Utils::parseMac(flow->core.dst_mac, value); - break; - case IPV4_SRC_ADDR: - case IPV6_SRC_ADDR: - /* - The following check prevents an empty ip address (e.g., ::) to - to overwrite another valid ip address already set. - This can happen for example when nProbe is configured (-T) to export - both %IPV4_SRC_ADDR and the %IPV6_SRC_ADDR. In that cases nProbe can - export a valid ipv4 and an empty ipv6. Without the check, the empty - v6 address may overwrite the non empty v4. - */ - if(flow->core.src_ip.isEmpty()) { - flow->core.src_ip.set((char*)value); - } else { - ip_aux.set((char*)value); - if(!ip_aux.isEmpty() && !ntop->getPrefs()->do_override_src_with_post_nat_src()) - /* tried to overwrite a non-empty IP with another non-empty IP */ - ntop->getTrace()->traceEvent(TRACE_WARNING, - "Attempt to set source ip multiple times. " - "Check exported fields"); - } - break; - case IP_PROTOCOL_VERSION: - flow->core.version = atoi(value); - break; - - case IPV4_DST_ADDR: - case IPV6_DST_ADDR: - if(flow->core.dst_ip.isEmpty()) { - flow->core.dst_ip.set((char*)value); - } else { - ip_aux.set((char*)value); - if(!ip_aux.isEmpty() && !ntop->getPrefs()->do_override_dst_with_post_nat_dst()) - ntop->getTrace()->traceEvent(TRACE_WARNING, - "Attempt to set destination ip multiple times. " - "Check exported fields"); - } - break; - case L4_SRC_PORT: - if(!flow->core.src_port) flow->core.src_port = htons(atoi(value)); - break; - case L4_DST_PORT: - if(!flow->core.dst_port) flow->core.dst_port = htons(atoi(value)); - break; - case SRC_VLAN: - case DST_VLAN: - flow->core.vlan_id = atoi(value); - break; - case DOT1Q_SRC_VLAN: - case DOT1Q_DST_VLAN: - if (flow->core.vlan_id == 0) - /* as those fields are the outer vlans in q-in-q - we set the vlan_id only if there is no inner vlan - value set - */ - flow->core.vlan_id = atoi(value); - break; - case PROTOCOL: - flow->core.l4_proto = atoi(value); - break; - case TCP_FLAGS: - flow->core.tcp.tcp_flags = atoi(value); - break; - case INITIATOR_PKTS: - flow->core.absolute_packet_octet_counters = true; - /* Don't break */ - case IN_PKTS: - flow->core.in_pkts = atol(value); - break; - case INITIATOR_OCTETS: - flow->core.absolute_packet_octet_counters = true; - /* Don't break */ - case IN_BYTES: - flow->core.in_bytes = atol(value); - break; - case RESPONDER_PKTS: - flow->core.absolute_packet_octet_counters = true; - /* Don't break */ - case OUT_PKTS: - flow->core.out_pkts = atol(value); - break; - case RESPONDER_OCTETS: - flow->core.absolute_packet_octet_counters = true; - /* Don't break */ - case OUT_BYTES: - flow->core.out_bytes = atol(value); - break; - case FIRST_SWITCHED: - flow->core.first_switched = atol(value); - break; - case LAST_SWITCHED: - flow->core.last_switched = atol(value); - break; - case SAMPLING_INTERVAL: - flow->core.pkt_sampling_rate = atoi(value); - break; - case DIRECTION: - flow->core.direction = atoi(value); - break; - case EXPORTER_IPV4_ADDRESS: - /* Format: a.b.c.d, possibly overrides NPROBE_IPV4_ADDRESS */ - if(ntohl(inet_addr(value)) && (flow->core.deviceIP = ntohl(inet_addr(value)))) - return false; - break; - case INPUT_SNMP: - flow->core.inIndex = atoi(value); - break; - case OUTPUT_SNMP: - flow->core.outIndex = atoi(value); - break; - case POST_NAT_SRC_IPV4_ADDR: - if(ntop->getPrefs()->do_override_src_with_post_nat_src()) { - IpAddress ip; - - ip.set((char*)value); - memcpy(&flow->core.src_ip, ip.getIP(), sizeof(flow->core.src_ip)); - } - break; - case POST_NAT_DST_IPV4_ADDR: - if(ntop->getPrefs()->do_override_dst_with_post_nat_dst()) { - IpAddress ip; - - ip.set((char*)value); - memcpy(&flow->core.dst_ip, ip.getIP(), sizeof(flow->core.dst_ip)); - } - break; - case POST_NAPT_SRC_TRANSPORT_PORT: - if(ntop->getPrefs()->do_override_src_with_post_nat_src()) - flow->core.src_port = htons(atoi(value)); - break; - case POST_NAPT_DST_TRANSPORT_PORT: - if(ntop->getPrefs()->do_override_dst_with_post_nat_dst()) - flow->core.dst_port = htons(atoi(value)); - break; - case INGRESS_VRFID: - flow->core.vrfId = atoi(value); - break; - case IPV4_SRC_MASK: - case IPV4_DST_MASK: - if(strcmp(value, "0")) - return false; - break; - case IPV4_NEXT_HOP: - if(strcmp(value, "0.0.0.0")) - return false; - break; - - default: - return false; - } - - return true; -} - -/* **************************************************** */ - -bool ParserInterface::parsePENNtopField(ZMQ_Flow * const flow, u_int32_t field, const char * const value) const { - switch(field) { - case L7_PROTO: - if(!strchr(value, '.')) { - /* Old behaviour, only the app protocol */ - flow->core.l7_proto.app_protocol = atoi(value); - } else { - char *proto_dot; - - flow->core.l7_proto.master_protocol = (u_int16_t)strtoll(value, &proto_dot, 10); - flow->core.l7_proto.app_protocol = (u_int16_t)strtoll(proto_dot + 1, NULL, 10); - } - -#if 0 - ntop->getTrace()->traceEvent(TRACE_NORMAL, "[value: %s][master: %u][app: %u]", - value, - flow->core.l7_proto.master_protocol, - flow->core.l7_proto.app_protocol); -#endif - break; - case OOORDER_IN_PKTS: - flow->core.tcp.ooo_in_pkts = atol(value); - break; - case OOORDER_OUT_PKTS: - flow->core.tcp.ooo_out_pkts = atol(value); - break; - case RETRANSMITTED_IN_PKTS: - flow->core.tcp.retr_in_pkts = atol(value); - break; - case RETRANSMITTED_OUT_PKTS: - flow->core.tcp.retr_out_pkts = atol(value); - break; - /* TODO add lost in/out to nProbe and here */ - case CLIENT_NW_LATENCY_MS: - { - float client_nw_latency = atof(value); - flow->core.tcp.clientNwLatency.tv_sec = client_nw_latency / 1e3; - flow->core.tcp.clientNwLatency.tv_usec = 1e3 * (client_nw_latency - flow->core.tcp.clientNwLatency.tv_sec * 1e3); - break; - } - case SERVER_NW_LATENCY_MS: - { - float server_nw_latency = atof(value); - flow->core.tcp.serverNwLatency.tv_sec = server_nw_latency / 1e3; - flow->core.tcp.serverNwLatency.tv_usec = 1e3 * (server_nw_latency - flow->core.tcp.serverNwLatency.tv_sec * 1e3); - break; - } - case CLIENT_TCP_FLAGS: - flow->core.tcp.client_tcp_flags = atoi(value); - case SERVER_TCP_FLAGS: - flow->core.tcp.server_tcp_flags = atoi(value); - case APPL_LATENCY_MS: - flow->core.tcp.applLatencyMsec = atof(value); - break; - case DNS_QUERY: - flow->dns_query = strdup(value); - break; - case HTTP_URL: - flow->http_url = strdup(value); - break; - case HTTP_SITE: - flow->http_site = strdup(value); - break; - case SSL_SERVER_NAME: - flow->ssl_server_name = strdup(value); - break; - case BITTORRENT_HASH: - flow->bittorrent_hash = strdup(value); - break; - case NPROBE_IPV4_ADDRESS: - /* Do not override EXPORTER_IPV4_ADDRESS */ - if(flow->core.deviceIP == 0 && (flow->core.deviceIP = ntohl(inet_addr(value)))) - return false; - break; - default: - return false; - } - - return true; -} - -/* **************************************************** */ - -void ParserInterface::parseSingleFlow(json_object *o, - u_int8_t source_id, - NetworkInterface *iface) { - ZMQ_Flow flow; - IpAddress ip_aux; /* used to check empty IPs */ - struct json_object_iterator it = json_object_iter_begin(o); - struct json_object_iterator itEnd = json_object_iter_end(o); - bool invalid_flow = false; - - /* Reset data */ - memset(&flow, 0, sizeof(flow)); - flow.core.l7_proto.master_protocol = flow.core.l7_proto.app_protocol = NDPI_PROTOCOL_UNKNOWN; - flow.core.l7_proto.category = NDPI_PROTOCOL_CATEGORY_UNSPECIFIED; - flow.additional_fields = json_object_new_object(); - flow.core.pkt_sampling_rate = 1; /* 1:1 (no sampling) */ - flow.core.source_id = source_id, flow.core.vlan_id = 0; - - while(!json_object_iter_equal(&it, &itEnd)) { - const char *key = json_object_iter_peek_name(&it); - json_object *v = json_object_iter_peek_value(&it); - const char *value = json_object_get_string(v); - bool add_to_additional_fields = false; - - if((key != NULL) && (value != NULL)) { - u_int32_t pen, key_id; - json_object *additional_o = json_tokener_parse(value); - bool res; - - getKeyId((char*)key, &pen, &key_id); - - switch(pen) { - case 0: /* No PEN */ - res = parsePENZeroField(&flow, key_id, value); - if(res) - break; - /* Dont'break when res == false for backward compatibility: attempt to parse Zero-PEN as Ntop-PEN */ - case NTOP_PEN: - res = parsePENNtopField(&flow, key_id, value); - break; - case UNKNOWN_PEN: - default: - res = false; - break; - } - - if(!res) { - switch(key_id) { - case 0: //json additional object added by Flow::serialize() - if((additional_o != NULL) && (strcmp(key,"json") == 0)) { - struct json_object_iterator additional_it = json_object_iter_begin(additional_o); - struct json_object_iterator additional_itEnd = json_object_iter_end(additional_o); - - while(!json_object_iter_equal(&additional_it, &additional_itEnd)) { - - const char *additional_key = json_object_iter_peek_name(&additional_it); - json_object *additional_v = json_object_iter_peek_value(&additional_it); - const char *additional_value = json_object_get_string(additional_v); - - if((additional_key != NULL) && (additional_value != NULL)) { - json_object_object_add(flow.additional_fields, additional_key, - json_object_new_string(additional_value)); - } - json_object_iter_next(&additional_it); - } - } - break; - default: -#ifdef NTOPNG_PRO - if(custom_app_maps || (custom_app_maps = new(std::nothrow) CustomAppMaps())) - custom_app_maps->checkCustomApp(key, value, &flow); -#endif - ntop->getTrace()->traceEvent(TRACE_DEBUG, "Not handled ZMQ field %u/%s", key_id, key); - add_to_additional_fields = true; - break; - } /* switch */ - } - - if(add_to_additional_fields) - json_object_object_add(flow.additional_fields, - key, json_object_new_string(value)); - - if(additional_o) json_object_put(additional_o); - } /* if */ - - /* Move to the next element */ - json_object_iter_next(&it); - } // while json_object_iter_equal - - if(flow.core.vlan_id && ntop->getPrefs()->do_ignore_vlans()) - flow.core.vlan_id = 0; - - /* Handle zero IPv4/IPv6 discrepacies */ - if(flow.core.version == 0) { - if(flow.core.src_ip.getVersion() != flow.core.dst_ip.getVersion()) { - if(flow.core.dst_ip.isIPv4() && flow.core.src_ip.isIPv6() && flow.core.src_ip.isEmpty()) - flow.core.src_ip.setVersion(4); - else if(flow.core.src_ip.isIPv4() && flow.core.dst_ip.isIPv6() && flow.core.dst_ip.isEmpty()) - flow.core.dst_ip.setVersion(4); - else if(flow.core.dst_ip.isIPv6() && flow.core.src_ip.isIPv4() && flow.core.src_ip.isEmpty()) - flow.core.src_ip.setVersion(6); - else if(flow.core.src_ip.isIPv6() && flow.core.dst_ip.isIPv4() && flow.core.dst_ip.isEmpty()) - flow.core.dst_ip.setVersion(6); - else { - invalid_flow = true; - ntop->getTrace()->traceEvent(TRACE_WARNING, - "IP version mismatch: client:%d server:%d - flow will be ignored", - flow.core.src_ip.getVersion(), flow.core.dst_ip.getVersion()); - } - } - } else - flow.core.src_ip.setVersion(flow.core.version), flow.core.dst_ip.setVersion(flow.core.version); - - if(!invalid_flow) { - /* Process Flow */ - iface->processFlow(&flow); - } - - /* Dispose memory */ - if(flow.dns_query) free(flow.dns_query); - if(flow.http_url) free(flow.http_url); - if(flow.http_site) free(flow.http_site); - if(flow.ssl_server_name) free(flow.ssl_server_name); - if(flow.bittorrent_hash) free(flow.bittorrent_hash); - - // json_object_put(o); - json_object_put(flow.additional_fields); -} - -/* **************************************************** */ - -u_int8_t ParserInterface::parseFlow(const char * const payload, int payload_size, u_int8_t source_id, void *data) { - json_object *f; - enum json_tokener_error jerr = json_tokener_success; - NetworkInterface *iface = (NetworkInterface*)data; - - // payload[payload_size] = '\0'; - // ntop->getTrace()->traceEvent(TRACE_NORMAL, "%s", payload); - - f = json_tokener_parse_verbose(payload, &jerr); - - if(f != NULL) { - int rc; - - if(json_object_get_type(f) == json_type_array) { - /* Flow array */ - int id, num_elements = json_object_array_length(f); - - for(id = 0; id < num_elements; id++) - parseSingleFlow(json_object_array_get_idx(f, id), source_id, iface); - - rc = num_elements; - } else { - parseSingleFlow(f, source_id, iface); - rc = 1; - } - - json_object_put(f); - return(rc); - } else { - // if o != NULL - if(!once) { - ntop->getTrace()->traceEvent(TRACE_WARNING, - "Invalid message received: your nProbe sender is outdated, data encrypted or invalid JSON?"); - ntop->getTrace()->traceEvent(TRACE_WARNING, "JSON Parse error [%s] payload size: %u payload: %s", - json_tokener_error_desc(jerr), - payload_size, - payload); - } - - once = true; - return 0; - } - - return 0; -} - -/* **************************************************** */ - -u_int8_t ParserInterface::parseCounter(const char * const payload, int payload_size, u_int8_t source_id, void *data) { - json_object *o; - enum json_tokener_error jerr = json_tokener_success; - NetworkInterface * iface = (NetworkInterface*)data; - sFlowInterfaceStats stats; - - // payload[payload_size] = '\0'; - - memset(&stats, 0, sizeof(stats)); - o = json_tokener_parse_verbose(payload, &jerr); - - if(o != NULL) { - struct json_object_iterator it = json_object_iter_begin(o); - struct json_object_iterator itEnd = json_object_iter_end(o); - - /* Reset data */ - memset(&stats, 0, sizeof(stats)); - - while(!json_object_iter_equal(&it, &itEnd)) { - const char *key = json_object_iter_peek_name(&it); - json_object *v = json_object_iter_peek_value(&it); - const char *value = json_object_get_string(v); - - if((key != NULL) && (value != NULL)) { - if(!strcmp(key, "deviceIP")) stats.deviceIP = ntohl(inet_addr(value)); - else if(!strcmp(key, "ifIndex")) stats.ifIndex = atol(value); - else if(!strcmp(key, "ifType")) stats.ifType = atol(value); - else if(!strcmp(key, "ifSpeed")) stats.ifSpeed = atol(value); - else if(!strcmp(key, "ifDirection")) stats.ifFullDuplex = (!strcmp(value, "Full")) ? true : false; - else if(!strcmp(key, "ifAdminStatus")) stats.ifAdminStatus = (!strcmp(value, "Up")) ? true : false; - else if(!strcmp(key, "ifOperStatus")) stats.ifOperStatus = (!strcmp(value, "Up")) ? true : false; - else if(!strcmp(key, "ifInOctets")) stats.ifInOctets = atoll(value); - else if(!strcmp(key, "ifInPackets")) stats.ifInPackets = atoll(value); - else if(!strcmp(key, "ifInErrors")) stats.ifInErrors = atoll(value); - else if(!strcmp(key, "ifOutOctets")) stats.ifOutOctets = atoll(value); - else if(!strcmp(key, "ifOutPackets")) stats.ifOutPackets = atoll(value); - else if(!strcmp(key, "ifOutErrors")) stats.ifOutErrors = atoll(value); - else if(!strcmp(key, "ifPromiscuousMode")) stats.ifPromiscuousMode = (!strcmp(value, "1")) ? true : false; - } /* if */ - - /* Move to the next element */ - json_object_iter_next(&it); - } // while json_object_iter_equal - - /* Process Flow */ - iface->processInterfaceStats(&stats); - - json_object_put(o); - } else { - // if o != NULL - if(!once) -{ ntop->getTrace()->traceEvent(TRACE_WARNING, - "Invalid message received: your nProbe sender is outdated, data encrypted or invalid JSON?"); - ntop->getTrace()->traceEvent(TRACE_WARNING, "JSON Parse error [%s] payload size: %u payload: %s", - json_tokener_error_desc(jerr), - payload_size, - payload); - } - once = true; - return -1; - } - - return 0; -} - -/* **************************************************** */ - -u_int8_t ParserInterface::parseTemplate(const char * const payload, int payload_size, u_int8_t source_id, void *data) { - /* The format that is currently defined for templates is a JSON as follows: - - [{"PEN":0,"field":1,"len":4,"format":"formatted_uint","name":"IN_BYTES","descr":"Incoming flow bytes (src->dst)"},{"PEN":0,"field":2,"len":4,"format":"formatted_uint","name":"IN_PKTS","descr":"Incoming flow packets (src->dst)"},] - */ - - // ntop->getTrace()->traceEvent(TRACE_NORMAL, "%s", payload); - - ZMQ_Template zmq_template; - json_object *obj, *w, *z; - enum json_tokener_error jerr = json_tokener_success; - - memset(&zmq_template, 0, sizeof(zmq_template)); - obj = json_tokener_parse_verbose(payload, &jerr); - - if(obj) { - if(json_object_get_type(obj) == json_type_array) { - int i, num_elements = json_object_array_length(obj); - - for(i = 0; i < num_elements; i++) { - memset(&zmq_template, 0, sizeof(zmq_template)); - - w = json_object_array_get_idx(obj, i); - - if(json_object_object_get_ex(w, "PEN", &z)) - zmq_template.pen = (u_int32_t)json_object_get_int(z); - - if(json_object_object_get_ex(w, "field", &z)) - zmq_template.field = (u_int32_t)json_object_get_int(z); - - if(json_object_object_get_ex(w, "format", &z)) - zmq_template.format = json_object_get_string(z); - - if(json_object_object_get_ex(w, "name", &z)) - zmq_template.name = json_object_get_string(z); - - if(json_object_object_get_ex(w, "descr", &z)) - zmq_template.descr = json_object_get_string(z); - - if(zmq_template.name) - addMapping(zmq_template.name, zmq_template.field, zmq_template.pen); - - // ntop->getTrace()->traceEvent(TRACE_NORMAL, "Template [PEN: %u][field: %u][format: %s][name: %s][descr: %s]", - // zmq_template.pen, zmq_template.field, zmq_template.format, zmq_template.name, zmq_template.descr) - ; - } - } - json_object_put(obj); - } else { - // if o != NULL - if(!once) { - ntop->getTrace()->traceEvent(TRACE_WARNING, - "Invalid message received: your nProbe sender is outdated, data encrypted or invalid JSON?"); - ntop->getTrace()->traceEvent(TRACE_WARNING, "JSON Parse error [%s] payload size: %u payload: %s", - json_tokener_error_desc(jerr), - payload_size, - payload); - } - once = true; - return -1; - } - - return 0; -} - -/* **************************************************** */ - -void ParserInterface::setFieldMap(const ZMQ_FieldMap * const field_map) const { - char hname[CONST_MAX_LEN_REDIS_KEY], key[32]; - snprintf(hname, sizeof(hname), CONST_FIELD_MAP_CACHE_KEY, get_id(), field_map->pen); - snprintf(key, sizeof(key), "%u", field_map->field); - - ntop->getRedis()->hashSet(hname, key, field_map->map); -} - -/* **************************************************** */ - -void ParserInterface::setFieldValueMap(const ZMQ_FieldValueMap * const field_value_map) const { - char hname[CONST_MAX_LEN_REDIS_KEY], key[32]; - snprintf(hname, sizeof(hname), CONST_FIELD_VALUE_MAP_CACHE_KEY, get_id(), field_value_map->pen, field_value_map->field); - snprintf(key, sizeof(key), "%u", field_value_map->value); - - ntop->getRedis()->hashSet(hname, key, field_value_map->map); -} - -/* **************************************************** */ - -u_int8_t ParserInterface::parseOptionFieldMap(json_object * const jo) const { - int arraylen = json_object_array_length(jo); - json_object *w, *z; - ZMQ_FieldMap field_map; - memset(&field_map, 0, sizeof(field_map)); - - for(int i = 0; i < arraylen; i++) { - w = json_object_array_get_idx(jo, i); - - if(json_object_object_get_ex(w, "PEN", &z)) - field_map.pen = (u_int32_t)json_object_get_int(z); - - if(json_object_object_get_ex(w, "field", &z)) { - field_map.field = (u_int32_t)json_object_get_int(z); - - if(json_object_object_get_ex(w, "map", &z)) { - field_map.map = json_object_to_json_string(z); - - setFieldMap(&field_map); - -#ifdef CUSTOM_APP_DEBUG - ntop->getTrace()->traceEvent(TRACE_NORMAL, "Option FieldMap [PEN: %u][field: %u][map: %s]", - field_map.pen, field_map.field, field_map.map); -#endif - } - } - } - - return 0; -} - -/* **************************************************** */ - -u_int8_t ParserInterface::parseOptionFieldValueMap(json_object * const w) const { - json_object *z; - ZMQ_FieldValueMap field_value_map; - memset(&field_value_map, 0, sizeof(field_value_map)); - - if(json_object_object_get_ex(w, "PEN", &z)) - field_value_map.pen = (u_int32_t)json_object_get_int(z); - - if(json_object_object_get_ex(w, "field", &z)) { - field_value_map.field = (u_int32_t)json_object_get_int(z); - - if(json_object_object_get_ex(w, "value", &z)) { - field_value_map.value = (u_int32_t)json_object_get_int(z); - - if(json_object_object_get_ex(w, "map", &z)) { - field_value_map.map = json_object_to_json_string(z); - - setFieldValueMap(&field_value_map); - -#ifdef CUSTOM_APP_DEBUG - ntop->getTrace()->traceEvent(TRACE_NORMAL, "Option FieldValueMap [PEN: %u][field: %u][value: %u][map: %s]", - field_value_map.pen, field_value_map.field, field_value_map.value, field_value_map.map); -#endif - } - } - } - - return 0; -} - -/* **************************************************** */ - -u_int8_t ParserInterface::parseOption(const char * const payload, int payload_size, u_int8_t source_id, void *data) { - /* The format that is currently defined for options is a JSON as follows: - - char opt[] = " - "{\"PEN\":8741, \"field\": 22, \"value\":1, \"map\":{\"name\":\"Skype\"}}," - "{\"PEN\":8741, \"field\": 22, \"value\":3, \"map\":{\"name\":\"Winni\"}}"; - - parseOption(opt, strlen(opt), source_id, this); - */ - - // ntop->getTrace()->traceEvent(TRACE_NORMAL, "%s", payload); - - json_object *o; - enum json_tokener_error jerr = json_tokener_success; - - o = json_tokener_parse_verbose(payload, &jerr); - - if(o != NULL) { - parseOptionFieldValueMap(o); - json_object_put(o); - } else { - // if o != NULL - if(!once) { - ntop->getTrace()->traceEvent(TRACE_WARNING, - "Invalid message received: your nProbe sender is outdated, data encrypted or invalid JSON?"); - ntop->getTrace()->traceEvent(TRACE_WARNING, "JSON Parse error [%s] payload size: %u payload: %s", - json_tokener_error_desc(jerr), - payload_size, - payload); - } - once = true; - return -1; - } - - return 0; -} - -/* **************************************** */ - -void ParserInterface::setRemoteStats(ZMQ_RemoteStats *zrs) { - if(!zrs) return; - - ifSpeed = zrs->remote_ifspeed, last_pkt_rcvd = 0, last_pkt_rcvd_remote = zrs->remote_time, - last_remote_pps = zrs->avg_pps, last_remote_bps = zrs->avg_bps; - - if((zmq_initial_pkts == 0) /* ntopng has been restarted */ - || (zrs->remote_bytes < zmq_initial_bytes) /* nProbe has been restarted */ - ) { - /* Start over */ - zmq_initial_bytes = zrs->remote_bytes, zmq_initial_pkts = zrs->remote_pkts; - } - - if(zmq_remote_initial_exported_flows == 0 /* ntopng has been restarted */ - || zrs->num_flow_exports < zmq_remote_initial_exported_flows) /* nProbe has been restarted */ - zmq_remote_initial_exported_flows = zrs->num_flow_exports; - - if(zmq_remote_stats_shadow) free(zmq_remote_stats_shadow); - zmq_remote_stats_shadow = zmq_remote_stats; - zmq_remote_stats = zrs; - - /* - * Don't override ethStats here, these stats are properly updated - * inside NetworkInterface::processFlow for ZMQ interfaces. - * Overriding values here may cause glitches and non-strictly-increasing counters - * yielding negative rates. - ethStats.setNumBytes(zrs->remote_bytes), ethStats.setNumPackets(zrs->remote_pkts); - * - */ -} - -/* **************************************************** */ - -#ifdef NTOPNG_PRO -bool ParserInterface::getCustomAppDetails(u_int32_t remapped_app_id, u_int32_t *const pen, u_int32_t *const app_field, u_int32_t *const app_id) { - return custom_app_maps && custom_app_maps->getCustomAppDetails(remapped_app_id, pen, app_field, app_id); -} -#endif - -/* **************************************************** */ - -void ParserInterface::lua(lua_State* vm) { - ZMQ_RemoteStats *zrs = zmq_remote_stats; - - NetworkInterface::lua(vm); - - if(zrs) { - if(zrs->remote_ifname[0] != '\0') - lua_push_str_table_entry(vm, "remote.name", zrs->remote_ifname); - if(zrs->remote_ifaddress[0] != '\0') - lua_push_str_table_entry(vm, "remote.if_addr",zrs->remote_ifaddress); - if(zrs->remote_probe_address[0] != '\0') - lua_push_str_table_entry(vm, "probe.ip", zrs->remote_probe_address); - if(zrs->remote_probe_public_address[0] != '\0') - lua_push_str_table_entry(vm, "probe.public_ip", zrs->remote_probe_public_address); - - lua_push_uint64_table_entry(vm, "zmq.num_flow_exports", zrs->num_flow_exports - zmq_remote_initial_exported_flows); - lua_push_uint64_table_entry(vm, "zmq.num_exporters", zrs->num_exporters); - - if(zrs->export_queue_full > 0) - lua_push_uint64_table_entry(vm, "zmq.drops.export_queue_full", zrs->export_queue_full); - lua_push_uint64_table_entry(vm, "zmq.drops.flow_collection_drops", zrs->flow_collection_drops); - lua_push_uint64_table_entry(vm, "timeout.lifetime", zrs->remote_lifetime_timeout); - lua_push_uint64_table_entry(vm, "timeout.idle", zrs->remote_idle_timeout); - } } /* **************************************************** */ diff --git a/src/SyslogCollectorInterface.cpp b/src/SyslogCollectorInterface.cpp new file mode 100644 index 000000000000..b207e5cbb9e1 --- /dev/null +++ b/src/SyslogCollectorInterface.cpp @@ -0,0 +1,368 @@ +/* + * + * (C) 2013-19 - ntop.org + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + */ + +#include "ntop_includes.h" + +#ifndef HAVE_NEDGE + +/* **************************************************** */ + +SyslogCollectorInterface::SyslogCollectorInterface(const char *_endpoint) : SyslogParserInterface(_endpoint) { + char *tmp, *port_ptr, *server_address; + int server_port; + int reuse = 1; + int i; + + endpoint = strdup(_endpoint); + + if (endpoint == NULL) + throw("memory allocation error"); + + tmp = strdup(_endpoint); + + if (tmp == NULL) + throw("memory allocation error"); + + if (strncmp(tmp, (char*) "syslog://", 9) == 0) { + server_address = &tmp[9]; + } else { + server_address = tmp; + } + + port_ptr = strchr(server_address, ':'); + + if (port_ptr != NULL) { + port_ptr[0] = '\0'; + port_ptr++; + server_port = atoi(port_ptr); + } else { + throw("bad tcp bind address format"); + } + + ntop->getTrace()->traceEvent(TRACE_NORMAL, "Starting TCP collector on %s:%d", + server_address, server_port); + + listen_sock = socket(AF_INET, SOCK_STREAM, 0); + + if (listen_sock < 0) + throw("socket error"); + + /* Allow to re-bind in case previous instance died */ + if (setsockopt(listen_sock, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)) != 0) + throw("setsockopt error"); + + memset(&listen_addr, 0, sizeof(listen_addr)); + + listen_addr.sin_family = AF_INET; + listen_addr.sin_addr.s_addr = inet_addr(server_address); + listen_addr.sin_port = htons(server_port); + + if (bind(listen_sock, (struct sockaddr *) &listen_addr, sizeof(struct sockaddr)) != 0) + throw("bind error"); + + if (listen(listen_sock, MAX_ZMQ_SUBSCRIBERS) != 0) + throw("listen error"); + + ntop->getTrace()->traceEvent(TRACE_NORMAL, "Accepting connections on %s:%d", + server_address, server_port); + + for (i = 0; i < MAX_ZMQ_SUBSCRIBERS; ++i) + connections[i].socket = 0; + + free(tmp); +} + +/* **************************************************** */ + +SyslogCollectorInterface::~SyslogCollectorInterface() { + int i; + + close(listen_sock); + + for (i = 0; i < MAX_ZMQ_SUBSCRIBERS; ++i) + if (connections[i].socket != 0) + close(connections[i].socket); + + free(endpoint); +} + +/* **************************************************** */ + +int SyslogCollectorInterface::initFDSets(fd_set *read_fds, fd_set *write_fds, fd_set *except_fds) { + int i; + + FD_ZERO(read_fds); + FD_ZERO(write_fds); + FD_ZERO(except_fds); + + FD_SET(listen_sock, read_fds); + FD_SET(listen_sock, except_fds); + + for (i = 0; i < MAX_ZMQ_SUBSCRIBERS; ++i) { + if (connections[i].socket != 0) { + FD_SET(connections[i].socket, read_fds); + FD_SET(connections[i].socket, except_fds); + } + } + + return 0; +} + +/* **************************************************** */ + +int SyslogCollectorInterface::handleNewConnection() { + char client_ipv4_str[INET_ADDRSTRLEN]; + struct sockaddr_in client_addr; + socklen_t client_len = sizeof(client_addr); + int new_client_sock; + int i; + + memset(&client_addr, 0, sizeof(client_addr)); + + new_client_sock = accept(listen_sock, (struct sockaddr *) &client_addr, &client_len); + + if (new_client_sock < 0) { + ntop->getTrace()->traceEvent(TRACE_ERROR, "accept() failure"); + return -1; + } + + inet_ntop(AF_INET, &client_addr.sin_addr, client_ipv4_str, INET_ADDRSTRLEN); + + ntop->getTrace()->traceEvent(TRACE_NORMAL, "Incoming connection from %s:%d", client_ipv4_str, client_addr.sin_port); + + for (i = 0; i < MAX_ZMQ_SUBSCRIBERS; ++i) { + if (connections[i].socket == 0) { + connections[i].socket = new_client_sock; + connections[i].address = client_addr; + return 0; + } + } + + ntop->getTrace()->traceEvent(TRACE_NORMAL, "Too many connections. Closing connection from %s:%d", + client_ipv4_str, client_addr.sin_port); + + close(new_client_sock); + + return -1; +} + +/* **************************************************** */ + +char *SyslogCollectorInterface::clientAddr2Str(syslog_client *client, char *buff) { + char client_ipv4_str[INET_ADDRSTRLEN]; + + inet_ntop(AF_INET, &client->address.sin_addr, client_ipv4_str, INET_ADDRSTRLEN); + + sprintf(buff, "%s:%d", client_ipv4_str, client->address.sin_port); + + return buff; +} + +/* **************************************************** */ + +void SyslogCollectorInterface::closeConnection(syslog_client *client) { + char buff[INET_ADDRSTRLEN + 10]; + + ntop->getTrace()->traceEvent(TRACE_NORMAL, "Closing client socket for %s\n", + clientAddr2Str(client, buff)); + + close(client->socket); + client->socket = 0; +} + +/* **************************************************** */ + +int SyslogCollectorInterface::receiveFromClient(syslog_client *client) { + char buffer[8192]; + char buff[INET_ADDRSTRLEN + 10]; + int len, received_total = 0; + int buffer_size = sizeof(buffer) - 1; + char *line, *pos; + + ntop->getTrace()->traceEvent(TRACE_INFO, "Trying to receive from %s", + clientAddr2Str(client, buff)); + + do { + + len = recv(client->socket, (char *) buffer, buffer_size, MSG_DONTWAIT); + + if (len < 0) { + if (errno == EAGAIN || errno == EWOULDBLOCK) { + ntop->getTrace()->traceEvent(TRACE_INFO, "Client is not ready"); + break; + } else { + ntop->getTrace()->traceEvent(TRACE_ERROR, "Client error"); + return -1; + } + + } else if (len == 0) { + ntop->getTrace()->traceEvent(TRACE_NORMAL, "Client shutdown"); + return -1; + + } else if (len > 0) { + received_total += len; + buffer[len] = '\0'; + line = strtok_r(buffer, "\n", &pos); + while (line) { + recvStats.num_flows += parseLog(line , this); + line = strtok_r(NULL, "\n", &pos); + } + } + + } while (len > 0); + + ntop->getTrace()->traceEvent(TRACE_INFO, "Total received bytes: %u", received_total); + + return 0; +} + +/* **************************************************** */ + +void SyslogCollectorInterface::collect_flows() { + u_int32_t max_num_polls_before_purge = MAX_ZMQ_POLLS_BEFORE_PURGE; + fd_set read_fds, write_fds, except_fds; + time_t now, next_purge_idle = time(NULL) + FLOW_PURGE_FREQUENCY;; + int high_sock = listen_sock; + int i, rc; + + ntop->getTrace()->traceEvent(TRACE_NORMAL, "Collecting flows on %s", ifname); + + while (isRunning()) { + while(idle()) { + purgeIdle(time(NULL)); + sleep(1); + if(ntop->getGlobals()->isShutdown()) return; + } + + initFDSets(&read_fds, &write_fds, &except_fds); + + high_sock = listen_sock; + for (i = 0; i < MAX_ZMQ_SUBSCRIBERS; i++) { + if (connections[i].socket > high_sock) + high_sock = connections[i].socket; + } + + rc = select(high_sock + 1, &read_fds, &write_fds, &except_fds, NULL); + + now = time(NULL); + max_num_polls_before_purge--; + if(rc == 0 || now >= next_purge_idle || max_num_polls_before_purge == 0) { + purgeIdle(now); + next_purge_idle = now + FLOW_PURGE_FREQUENCY; + max_num_polls_before_purge = MAX_ZMQ_POLLS_BEFORE_PURGE; + } + + if (rc <= 0) { + ntop->getTrace()->traceEvent(TRACE_ERROR, "select returned %d", rc); + } else { + + if (FD_ISSET(listen_sock, &read_fds)) { + handleNewConnection(); + } + + if (FD_ISSET(listen_sock, &except_fds)) { + ntop->getTrace()->traceEvent(TRACE_ERROR, "Exception on listen socket fd"); + } + + for (i = 0; i < MAX_ZMQ_SUBSCRIBERS; ++i) { + if (connections[i].socket != 0 && FD_ISSET(connections[i].socket, &read_fds)) { + if (receiveFromClient(&connections[i]) != 0) { + closeConnection(&connections[i]); + continue; + } + } + + if (connections[i].socket != 0 && FD_ISSET(connections[i].socket, &except_fds)) { + ntop->getTrace()->traceEvent(TRACE_ERROR, "Exception on client fd"); + closeConnection(&connections[i]); + continue; + } + } + } + } + + ntop->getTrace()->traceEvent(TRACE_NORMAL, "Flow collection is over."); +} + +/* **************************************************** */ + +static void* packetPollLoop(void* ptr) { + SyslogCollectorInterface *iface = (SyslogCollectorInterface*)ptr; + + /* Wait until the initialization completes */ + while(!iface->isRunning()) sleep(1); + + iface->collect_flows(); + return(NULL); +} + +/* **************************************************** */ + +void SyslogCollectorInterface::startPacketPolling() { + pthread_create(&pollLoop, NULL, packetPollLoop, (void*)this); + pollLoopCreated = true; + NetworkInterface::startPacketPolling(); +} + +/* **************************************************** */ + +void SyslogCollectorInterface::shutdown() { + void *res; + + if(running) { + NetworkInterface::shutdown(); + pthread_join(pollLoop, &res); + } +} + +/* **************************************************** */ + +bool SyslogCollectorInterface::set_packet_filter(char *filter) { + ntop->getTrace()->traceEvent(TRACE_ERROR, + "No filter can be set on a collector interface. Ignored %s", filter); + return(false); +} + +/* **************************************************** */ + +void SyslogCollectorInterface::lua(lua_State* vm) { + SyslogParserInterface::lua(vm); + + lua_newtable(vm); + lua_push_uint64_table_entry(vm, "flows", recvStats.num_flows); + lua_pushstring(vm, "syslogRecvStats"); + lua_insert(vm, -2); + lua_settable(vm, -3); +} + +/* **************************************************** */ + +void SyslogCollectorInterface::purgeIdle(time_t when) { + NetworkInterface::purgeIdle(when); + + if(flowHashing) { + FlowHashing *current, *tmp; + HASH_ITER(hh, flowHashing, current, tmp) + static_cast(current->iface)->purgeIdle(when); + } +} + +#endif diff --git a/src/SyslogParserInterface.cpp b/src/SyslogParserInterface.cpp new file mode 100644 index 000000000000..5740e8bb0c66 --- /dev/null +++ b/src/SyslogParserInterface.cpp @@ -0,0 +1,134 @@ +/* + * + * (C) 2019 - ntop.org + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + */ + +#include "ntop_includes.h" + +#ifndef HAVE_NEDGE + +//#define SYSLOG_DEBUG + +/* **************************************************** */ + +SyslogParserInterface::SyslogParserInterface(const char *endpoint, const char *custom_interface_type) : ParserInterface(endpoint, custom_interface_type) { + +} + +/* **************************************************** */ + +SyslogParserInterface::~SyslogParserInterface() { + +} + +/* **************************************************** */ + +u_int8_t SyslogParserInterface::parseLog(char *log_line, void *data) { + NetworkInterface *iface = (NetworkInterface*)data; + char *content; + enum json_tokener_error jerr = json_tokener_success; + int num_flows = 0; + + content = strstr(log_line, ": "); + + if (content == NULL) + return 0; /* unexpected format */ + + content[1] = '\0'; + content += 2; + + if (strstr(log_line, "suricata") != NULL) { + json_object *o; + Parsed_Flow flow; + + /* Suricata Log */ + +#ifdef SYSLOG_DEBUG + //ntop->getTrace()->traceEvent(TRACE_NORMAL, "[SYSLOG] Suricata EVE JSON: %s", content); +#endif + + /* Reset data */ + memset(&flow, 0, sizeof(flow)); + flow.core.l7_proto.master_protocol = flow.core.l7_proto.app_protocol = NDPI_PROTOCOL_UNKNOWN; + flow.core.l7_proto.category = NDPI_PROTOCOL_CATEGORY_UNSPECIFIED; + flow.additional_fields = json_object_new_object(); + flow.core.pkt_sampling_rate = 1; /* 1:1 (no sampling) */ + flow.core.source_id = 0, flow.core.vlan_id = 0; + + + o = json_tokener_parse_verbose(content, &jerr); + + if (o) { + json_object *w, *f; + +#ifdef SYSLOG_DEBUG + if (json_object_object_get_ex(o, "timestamp", &w)) + ntop->getTrace()->traceEvent(TRACE_NORMAL, "Suricata event timestamp: %s", json_object_get_string(w)); +#endif + + //if (json_object_object_get_ex(o, "flow_id", &w)) flow_id = json_object_get_string(w); + if (json_object_object_get_ex(o, "vlan", &w)) flow.core.vlan_id = json_object_get_int(w); + if (json_object_object_get_ex(o, "src_ip", &w)) flow.core.src_ip.set((char *) json_object_get_string(w)); + if (json_object_object_get_ex(o, "dest_ip", &w)) flow.core.dst_ip.set((char *) json_object_get_string(w)); + if (json_object_object_get_ex(o, "src_port", &w)) flow.core.src_port = htons(json_object_get_int(w)); + if (json_object_object_get_ex(o, "dest_port", &w)) flow.core.dst_port = htons(json_object_get_int(w)); + if (json_object_object_get_ex(o, "proto", &w)) flow.core.l4_proto = Utils::l4name2proto((char *) json_object_get_string(w)); + + if (json_object_object_get_ex(o, "flow", &f)) { + +#ifdef SYSLOG_DEBUG + if (json_object_object_get_ex(f, "start", &w)) + ntop->getTrace()->traceEvent(TRACE_NORMAL, "Suricata flow start: %s", json_object_get_string(w)); +#endif + + if (json_object_object_get_ex(f, "pkts_toserver", &w)) flow.core.in_pkts = json_object_get_int(w); + if (json_object_object_get_ex(f, "pkts_toclient", &w)) flow.core.out_pkts = json_object_get_int(w); + if (json_object_object_get_ex(f, "bytes_toserver", &w)) flow.core.in_bytes = json_object_get_int(w); + if (json_object_object_get_ex(f, "bytes_client", &w)) flow.core.out_bytes = json_object_get_int(w); + + iface->processFlow(&flow); + } + + json_object_put(o); + } + + if (flow.additional_fields) + json_object_put(flow.additional_fields); + + } else { + /* System Log */ +#ifdef SYSLOG_DEBUG + ntop->getTrace()->traceEvent(TRACE_INFO, "[SYSLOG] System Event (%s): %s", log_line, content); +#endif + } + + return num_flows; +} + +/* **************************************************** */ + +void SyslogParserInterface::lua(lua_State* vm) { + + NetworkInterface::lua(vm); + +} + +/* **************************************************** */ + +#endif diff --git a/src/TimeseriesStats.cpp b/src/TimeseriesStats.cpp index b0307bffd13a..91eac72079de 100644 --- a/src/TimeseriesStats.cpp +++ b/src/TimeseriesStats.cpp @@ -84,5 +84,6 @@ void TimeseriesStats::luaStats(lua_State* vm, NetworkInterface *iface, bool host lua_push_uint64_table_entry(vm, "other_ip.bytes.rcvd", other_ip_rcvd.getNumBytes()); host->luaDNS(vm); + host->luaTCP(vm); } } diff --git a/src/Utils.cpp b/src/Utils.cpp index 5f34f8b3daf5..91b263005c9d 100755 --- a/src/Utils.cpp +++ b/src/Utils.cpp @@ -155,6 +155,27 @@ char* Utils::l4proto2name(u_int8_t proto) { /* ****************************************************** */ +u_int8_t Utils::l4name2proto(char *name) { + if (strcmp(name, "IP") == 0) return 0; + else if (strcmp(name, "ICMP") == 0) return 1; + else if (strcmp(name, "IGMP") == 0) return 2; + else if (strcmp(name, "TCP") == 0) return 6; + else if (strcmp(name, "UDP") == 0) return 17; + else if (strcmp(name, "IPv6") == 0) return 41; + else if (strcmp(name, "RSVP") == 0) return 46; + else if (strcmp(name, "GRE") == 0) return 47; + else if (strcmp(name, "ESP") == 0) return 50; + else if (strcmp(name, "AH") == 0) return 51; + else if (strcmp(name, "IPv6-ICMP") == 0) return 58; + else if (strcmp(name, "OSPF") == 0) return 89; + else if (strcmp(name, "PIM") == 0) return 103; + else if (strcmp(name, "VRRP") == 0) return 112; + else if (strcmp(name, "HIP") == 0) return 139; + else return 0; +} + +/* ****************************************************** */ + #ifdef NOTUSED bool Utils::isIPAddress(char *ip) { struct in_addr addr4; @@ -3657,3 +3678,16 @@ void Utils::freeAuthenticator(HTTPAuthenticator *auth) { } /* ****************************************************** */ + +DetailsLevel Utils::bool2DetailsLevel(bool max, bool higher, bool normal){ + if(max){ + return details_max; + } else if(higher){ + return details_higher; + } else if(normal){ + return details_normal; + } + else{ + return details_high; + } +} diff --git a/src/ZCCollectorInterface.cpp b/src/ZCCollectorInterface.cpp index afe1c3587031..16e86595e47e 100644 --- a/src/ZCCollectorInterface.cpp +++ b/src/ZCCollectorInterface.cpp @@ -27,7 +27,7 @@ /* **************************************************** */ -ZCCollectorInterface::ZCCollectorInterface(const char *name) : ParserInterface(name) { +ZCCollectorInterface::ZCCollectorInterface(const char *name) : ZMQParserInterface(name) { char ifname[32]; char *at; diff --git a/src/CollectorInterface.cpp b/src/ZMQCollectorInterface.cpp similarity index 95% rename from src/CollectorInterface.cpp rename to src/ZMQCollectorInterface.cpp index ffe286382bdc..8f2d9e8c2769 100644 --- a/src/CollectorInterface.cpp +++ b/src/ZMQCollectorInterface.cpp @@ -25,7 +25,7 @@ /* **************************************************** */ -CollectorInterface::CollectorInterface(const char *_endpoint) : ParserInterface(_endpoint) { +ZMQCollectorInterface::ZMQCollectorInterface(const char *_endpoint) : ZMQParserInterface(_endpoint) { char *tmp, *e, *t; const char *topics[] = { "flow", "event", "counter", "template", "option", NULL }; @@ -125,7 +125,7 @@ CollectorInterface::CollectorInterface(const char *_endpoint) : ParserInterface( /* **************************************************** */ -CollectorInterface::~CollectorInterface() { +ZMQCollectorInterface::~ZMQCollectorInterface() { for(int i=0; iisRunning()) sleep(1); @@ -303,7 +303,7 @@ static void* packetPollLoop(void* ptr) { /* **************************************************** */ -void CollectorInterface::startPacketPolling() { +void ZMQCollectorInterface::startPacketPolling() { pthread_create(&pollLoop, NULL, packetPollLoop, (void*)this); pollLoopCreated = true; NetworkInterface::startPacketPolling(); @@ -311,7 +311,7 @@ void CollectorInterface::startPacketPolling() { /* **************************************************** */ -void CollectorInterface::shutdown() { +void ZMQCollectorInterface::shutdown() { void *res; if(running) { @@ -322,7 +322,7 @@ void CollectorInterface::shutdown() { /* **************************************************** */ -bool CollectorInterface::set_packet_filter(char *filter) { +bool ZMQCollectorInterface::set_packet_filter(char *filter) { ntop->getTrace()->traceEvent(TRACE_ERROR, "No filter can be set on a collector interface. Ignored %s", filter); return(false); @@ -330,8 +330,8 @@ bool CollectorInterface::set_packet_filter(char *filter) { /* **************************************************** */ -void CollectorInterface::lua(lua_State* vm) { - ParserInterface::lua(vm); +void ZMQCollectorInterface::lua(lua_State* vm) { + ZMQParserInterface::lua(vm); lua_newtable(vm); lua_push_uint64_table_entry(vm, "flows", recvStats.num_flows); @@ -345,7 +345,7 @@ void CollectorInterface::lua(lua_State* vm) { /* **************************************************** */ -void CollectorInterface::purgeIdle(time_t when) { +void ZMQCollectorInterface::purgeIdle(time_t when) { NetworkInterface::purgeIdle(when); if(flowHashing) { diff --git a/src/ZMQParserInterface.cpp b/src/ZMQParserInterface.cpp new file mode 100755 index 000000000000..e3230d7bb357 --- /dev/null +++ b/src/ZMQParserInterface.cpp @@ -0,0 +1,1028 @@ +/* + * + * (C) 2013-19 - ntop.org + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + */ + +#include "ntop_includes.h" + +#ifndef HAVE_NEDGE + +/* **************************************************** */ + +/* IMPORTANT: keep it in sync with flow_fields_description part of flow_utils.lua */ +ZMQParserInterface::ZMQParserInterface(const char *endpoint, const char *custom_interface_type) : ParserInterface(endpoint, custom_interface_type) { + zmq_initial_bytes = 0, zmq_initial_pkts = 0; + zmq_remote_stats = zmq_remote_stats_shadow = NULL; + zmq_remote_initial_exported_flows = 0; + once = false; +#ifdef NTOPNG_PRO + custom_app_maps = NULL; +#endif + + /* Populate defaults for @NTOPNG@ nProbe templates. No need to populate + all the fields as nProbe will sent them periodically. + + This minimum set is required for backward compatibility. */ + addMapping("IN_SRC_MAC", IN_SRC_MAC); + addMapping("OUT_SRC_MAC", OUT_SRC_MAC); + addMapping("IN_DST_MAC", IN_DST_MAC); + addMapping("OUT_DST_MAC", OUT_DST_MAC); + addMapping("SRC_VLAN", SRC_VLAN); + addMapping("DST_VLAN", DST_VLAN); + addMapping("DOT1Q_SRC_VLAN", DOT1Q_SRC_VLAN); + addMapping("DOT1Q_DST_VLAN", DOT1Q_DST_VLAN); + addMapping("INPUT_SNMP", INPUT_SNMP); + addMapping("OUTPUT_SNMP", OUTPUT_SNMP); + addMapping("IPV4_SRC_ADDR", IPV4_SRC_ADDR); + addMapping("IPV4_DST_ADDR", IPV4_DST_ADDR); + addMapping("L4_SRC_PORT", L4_SRC_PORT); + addMapping("L4_DST_PORT", L4_DST_PORT); + addMapping("IPV6_SRC_ADDR", IPV6_SRC_ADDR); + addMapping("IPV6_DST_ADDR", IPV6_DST_ADDR); + addMapping("IP_PROTOCOL_VERSION", IP_PROTOCOL_VERSION); + addMapping("PROTOCOL", PROTOCOL); + addMapping("L7_PROTO", L7_PROTO, NTOP_PEN); + addMapping("IN_BYTES", IN_BYTES); + addMapping("IN_PKTS", IN_PKTS); + addMapping("OUT_BYTES", OUT_BYTES); + addMapping("OUT_PKTS", OUT_PKTS); + addMapping("FIRST_SWITCHED", FIRST_SWITCHED); + addMapping("LAST_SWITCHED", LAST_SWITCHED); + addMapping("EXPORTER_IPV4_ADDRESS", EXPORTER_IPV4_ADDRESS); + addMapping("EXPORTER_IPV6_ADDRESS", EXPORTER_IPV6_ADDRESS); + addMapping("NPROBE_IPV4_ADDRESS", NPROBE_IPV4_ADDRESS, NTOP_PEN); + addMapping("TCP_FLAGS", TCP_FLAGS); + addMapping("INITIATOR_PKTS", INITIATOR_PKTS); + addMapping("INITIATOR_OCTETS", INITIATOR_OCTETS); + addMapping("RESPONDER_PKTS", RESPONDER_PKTS); + addMapping("RESPONDER_OCTETS", RESPONDER_OCTETS); + addMapping("SAMPLING_INTERVAL", SAMPLING_INTERVAL); + addMapping("DIRECTION", DIRECTION); + addMapping("POST_NAT_SRC_IPV4_ADDR", POST_NAT_SRC_IPV4_ADDR); + addMapping("POST_NAT_DST_IPV4_ADDR", POST_NAT_DST_IPV4_ADDR); + addMapping("POST_NAPT_SRC_TRANSPORT_PORT", POST_NAPT_SRC_TRANSPORT_PORT); + addMapping("POST_NAPT_DST_TRANSPORT_PORT", POST_NAPT_DST_TRANSPORT_PORT); + addMapping("INGRESS_VRFID", INGRESS_VRFID); + addMapping("IPV4_SRC_MASK", IPV4_SRC_MASK); + addMapping("IPV4_DST_MASK", IPV4_DST_MASK); + addMapping("IPV4_NEXT_HOP", IPV4_NEXT_HOP); + addMapping("OOORDER_IN_PKTS", OOORDER_IN_PKTS, NTOP_PEN); + addMapping("OOORDER_OUT_PKTS", OOORDER_OUT_PKTS, NTOP_PEN); + addMapping("RETRANSMITTED_IN_PKTS", RETRANSMITTED_IN_PKTS, NTOP_PEN); + addMapping("RETRANSMITTED_OUT_PKTS", RETRANSMITTED_OUT_PKTS, NTOP_PEN); + addMapping("DNS_QUERY", DNS_QUERY, NTOP_PEN); + addMapping("HTTP_URL", HTTP_URL, NTOP_PEN); + addMapping("HTTP_SITE", HTTP_SITE, NTOP_PEN); + addMapping("SSL_SERVER_NAME", SSL_SERVER_NAME, NTOP_PEN); + addMapping("BITTORRENT_HASH", BITTORRENT_HASH, NTOP_PEN); +} + +/* **************************************************** */ + +ZMQParserInterface::~ZMQParserInterface() { + if(zmq_remote_stats) free(zmq_remote_stats); + if(zmq_remote_stats_shadow) free(zmq_remote_stats_shadow); +#ifdef NTOPNG_PRO + if(custom_app_maps) delete(custom_app_maps); +#endif +} + +/* **************************************************** */ + +void ZMQParserInterface::addMapping(const char *sym, u_int32_t num, u_int32_t pen) { + string label(sym); + labels_map_t::iterator it; + + if((it = labels_map.find(label)) == labels_map.end()) + labels_map.insert(make_pair(label, make_pair(pen, num))); + else + it->second.first = pen, it->second.second = num; +} + +/* **************************************************** */ + +bool ZMQParserInterface::getKeyId(char *sym, u_int32_t * const pen, u_int32_t * const field) const { + u_int32_t cur_pen, cur_field; + string label(sym); + labels_map_t::const_iterator it; + + *pen = UNKNOWN_PEN, *field = UNKNOWN_FLOW_ELEMENT; + + if(sscanf(sym, "%u.%u", &cur_pen, &cur_field) == 2) + *pen = cur_pen, *field = cur_field; + else if(sscanf(sym, "%u", &cur_field) == 1) + *pen = 0, *field = cur_field; + else if((it = labels_map.find(label)) != labels_map.end()) + *pen = it->second.first, *field = it->second.second; + else + return false; + + return true; +} + +/* **************************************************** */ + +u_int8_t ZMQParserInterface::parseEvent(const char * const payload, int payload_size, + u_int8_t source_id, void *data) { + json_object *o; + enum json_tokener_error jerr = json_tokener_success; + NetworkInterface * iface = (NetworkInterface*)data; + ZMQ_RemoteStats *zrs = NULL; + memset((void*)&zrs, 0, sizeof(zrs)); + + // payload[payload_size] = '\0'; + + // ntop->getTrace()->traceEvent(TRACE_NORMAL, "%s", payload); + o = json_tokener_parse_verbose(payload, &jerr); + + if(o && (zrs = (ZMQ_RemoteStats*)calloc(1, sizeof(ZMQ_RemoteStats)))) { + json_object *w, *z; + + if(json_object_object_get_ex(o, "time", &w)) zrs->remote_time = (u_int32_t)json_object_get_int64(w); + if(json_object_object_get_ex(o, "bytes", &w)) zrs->remote_bytes = (u_int64_t)json_object_get_int64(w); + if(json_object_object_get_ex(o, "packets", &w)) zrs->remote_pkts = (u_int64_t)json_object_get_int64(w); + + if(json_object_object_get_ex(o, "iface", &w)) { + if(json_object_object_get_ex(w, "name", &z)) + snprintf(zrs->remote_ifname, sizeof(zrs->remote_ifname), "%s", json_object_get_string(z)); + if(json_object_object_get_ex(w, "speed", &z)) + zrs->remote_ifspeed = (u_int32_t)json_object_get_int64(z); + if(json_object_object_get_ex(w, "ip", &z)) + snprintf(zrs->remote_ifaddress, sizeof(zrs->remote_ifaddress), "%s", json_object_get_string(z)); + } + + if(json_object_object_get_ex(o, "probe", &w)) { + if(json_object_object_get_ex(w, "public_ip", &z)) + snprintf(zrs->remote_probe_public_address, sizeof(zrs->remote_probe_public_address), "%s", json_object_get_string(z)); + if(json_object_object_get_ex(w, "ip", &z)) + snprintf(zrs->remote_probe_address, sizeof(zrs->remote_probe_address), "%s", json_object_get_string(z)); + } + + if(json_object_object_get_ex(o, "avg", &w)) { + if(json_object_object_get_ex(w, "bps", &z)) + zrs->avg_bps = (u_int32_t)json_object_get_int64(z); + if(json_object_object_get_ex(w, "pps", &z)) + zrs->avg_pps = (u_int32_t)json_object_get_int64(z); + } + + if(json_object_object_get_ex(o, "timeout", &w)) { + if(json_object_object_get_ex(w, "lifetime", &z)) + zrs->remote_lifetime_timeout = (u_int32_t)json_object_get_int64(z); + if(json_object_object_get_ex(w, "idle", &z)) + zrs->remote_idle_timeout = (u_int32_t)json_object_get_int64(z); + } + + if(json_object_object_get_ex(o, "drops", &w)) { + if(json_object_object_get_ex(w, "export_queue_full", &z)) + zrs->export_queue_full = (u_int32_t)json_object_get_int64(z); + + if(json_object_object_get_ex(w, "too_many_flows", &z)) + zrs->too_many_flows = (u_int32_t)json_object_get_int64(z); + + if(json_object_object_get_ex(w, "elk_flow_drops", &z)) + zrs->elk_flow_drops = (u_int32_t)json_object_get_int64(z); + + if(json_object_object_get_ex(w, "sflow_pkt_sample_drops", &z)) + zrs->sflow_pkt_sample_drops = (u_int32_t)json_object_get_int64(z); + + if(json_object_object_get_ex(w, "flow_collection_drops", &z)) + zrs->flow_collection_drops = (u_int32_t)json_object_get_int64(z); + } + + if(json_object_object_get_ex(o, "zmq", &w)) { + if(json_object_object_get_ex(w, "num_flow_exports", &z)) + zrs->num_flow_exports = (u_int64_t)json_object_get_int64(z); + + if(json_object_object_get_ex(w, "num_exporters", &z)) + zrs->num_exporters = (u_int8_t)json_object_get_int(z); + } + +#ifdef ZMQ_EVENT_DEBUG + ntop->getTrace()->traceEvent(TRACE_NORMAL, "Event parsed " + "[iface: {name: %s, speed: %u, ip:%s}]" + "[probe: {public_ip: %s, ip: %s}]" + "[avg: {bps: %u, pps: %u}]" + "[remote: {time: %u, bytes: %u, packets: %u, idle_timeout: %u, lifetime_timeout:%u}]" + "[zmq: {num_exporters: %u, num_flow_exports: %u}]", + zrs->remote_ifname, zrs->remote_ifspeed, zrs->remote_ifaddress, + zrs->remote_probe_public_address, zrs->remote_probe_address, + zrs->avg_bps, zrs->avg_pps, + zrs->remote_time, (u_int32_t)zrs->remote_bytes, (u_int32_t)zrs->remote_pkts, + zrs->remote_idle_timeout, zrs->remote_lifetime_timeout, + zrs->num_exporters, zrs->num_flow_exports); +#endif + + /* ntop->getTrace()->traceEvent(TRACE_WARNING, "%u/%u", avg_bps, avg_pps); */ + + /* Process Flow */ + static_cast(iface)->setRemoteStats(zrs); + if(flowHashing) { + FlowHashing *current, *tmp; + + HASH_ITER(hh, flowHashing, current, tmp) { + ZMQ_RemoteStats *zrscopy = (ZMQ_RemoteStats*)malloc(sizeof(ZMQ_RemoteStats)); + + if(zrscopy) + memcpy(zrscopy, zrs, sizeof(ZMQ_RemoteStats)); + + static_cast(current->iface)->setRemoteStats(zrscopy); + } + } + + /* Dispose memory */ + json_object_put(o); + } else { + // if o != NULL + if(!once) { + ntop->getTrace()->traceEvent(TRACE_WARNING, + "Invalid message received: " + "your nProbe sender is outdated, data encrypted, invalid JSON, or oom?"); + ntop->getTrace()->traceEvent(TRACE_WARNING, "JSON Parse error [%s] payload size: %u payload: %s", + json_tokener_error_desc(jerr), + payload_size, + payload); + } + once = true; + if(o) json_object_put(o); + return -1; + } + + return 0; +} + +/* **************************************************** */ + +bool ZMQParserInterface::parsePENZeroField(Parsed_Flow * const flow, u_int32_t field, const char * const value) const { + IpAddress ip_aux; /* used to check empty IPs */ + + switch(field) { + case IN_SRC_MAC: + case OUT_SRC_MAC: + /* Format 00:00:00:00:00:00 */ + Utils::parseMac(flow->core.src_mac, value); + break; + case IN_DST_MAC: + case OUT_DST_MAC: + Utils::parseMac(flow->core.dst_mac, value); + break; + case IPV4_SRC_ADDR: + case IPV6_SRC_ADDR: + /* + The following check prevents an empty ip address (e.g., ::) to + to overwrite another valid ip address already set. + This can happen for example when nProbe is configured (-T) to export + both %IPV4_SRC_ADDR and the %IPV6_SRC_ADDR. In that cases nProbe can + export a valid ipv4 and an empty ipv6. Without the check, the empty + v6 address may overwrite the non empty v4. + */ + if(flow->core.src_ip.isEmpty()) { + flow->core.src_ip.set((char*)value); + } else { + ip_aux.set((char*)value); + if(!ip_aux.isEmpty() && !ntop->getPrefs()->do_override_src_with_post_nat_src()) + /* tried to overwrite a non-empty IP with another non-empty IP */ + ntop->getTrace()->traceEvent(TRACE_WARNING, + "Attempt to set source ip multiple times. " + "Check exported fields"); + } + break; + case IP_PROTOCOL_VERSION: + flow->core.version = atoi(value); + break; + + case IPV4_DST_ADDR: + case IPV6_DST_ADDR: + if(flow->core.dst_ip.isEmpty()) { + flow->core.dst_ip.set((char*)value); + } else { + ip_aux.set((char*)value); + if(!ip_aux.isEmpty() && !ntop->getPrefs()->do_override_dst_with_post_nat_dst()) + ntop->getTrace()->traceEvent(TRACE_WARNING, + "Attempt to set destination ip multiple times. " + "Check exported fields"); + } + break; + case L4_SRC_PORT: + if(!flow->core.src_port) flow->core.src_port = htons(atoi(value)); + break; + case L4_DST_PORT: + if(!flow->core.dst_port) flow->core.dst_port = htons(atoi(value)); + break; + case SRC_VLAN: + case DST_VLAN: + flow->core.vlan_id = atoi(value); + break; + case DOT1Q_SRC_VLAN: + case DOT1Q_DST_VLAN: + if (flow->core.vlan_id == 0) + /* as those fields are the outer vlans in q-in-q + we set the vlan_id only if there is no inner vlan + value set + */ + flow->core.vlan_id = atoi(value); + break; + case PROTOCOL: + flow->core.l4_proto = atoi(value); + break; + case TCP_FLAGS: + flow->core.tcp.tcp_flags = atoi(value); + break; + case INITIATOR_PKTS: + flow->core.absolute_packet_octet_counters = true; + /* Don't break */ + case IN_PKTS: + flow->core.in_pkts = atol(value); + break; + case INITIATOR_OCTETS: + flow->core.absolute_packet_octet_counters = true; + /* Don't break */ + case IN_BYTES: + flow->core.in_bytes = atol(value); + break; + case RESPONDER_PKTS: + flow->core.absolute_packet_octet_counters = true; + /* Don't break */ + case OUT_PKTS: + flow->core.out_pkts = atol(value); + break; + case RESPONDER_OCTETS: + flow->core.absolute_packet_octet_counters = true; + /* Don't break */ + case OUT_BYTES: + flow->core.out_bytes = atol(value); + break; + case FIRST_SWITCHED: + flow->core.first_switched = atol(value); + break; + case LAST_SWITCHED: + flow->core.last_switched = atol(value); + break; + case SAMPLING_INTERVAL: + flow->core.pkt_sampling_rate = atoi(value); + break; + case DIRECTION: + flow->core.direction = atoi(value); + break; + case EXPORTER_IPV4_ADDRESS: + /* Format: a.b.c.d, possibly overrides NPROBE_IPV4_ADDRESS */ + if(ntohl(inet_addr(value)) && (flow->core.deviceIP = ntohl(inet_addr(value)))) + return false; + break; + case INPUT_SNMP: + flow->core.inIndex = atoi(value); + break; + case OUTPUT_SNMP: + flow->core.outIndex = atoi(value); + break; + case POST_NAT_SRC_IPV4_ADDR: + if(ntop->getPrefs()->do_override_src_with_post_nat_src()) { + IpAddress ip; + + ip.set((char*)value); + memcpy(&flow->core.src_ip, ip.getIP(), sizeof(flow->core.src_ip)); + } + break; + case POST_NAT_DST_IPV4_ADDR: + if(ntop->getPrefs()->do_override_dst_with_post_nat_dst()) { + IpAddress ip; + + ip.set((char*)value); + memcpy(&flow->core.dst_ip, ip.getIP(), sizeof(flow->core.dst_ip)); + } + break; + case POST_NAPT_SRC_TRANSPORT_PORT: + if(ntop->getPrefs()->do_override_src_with_post_nat_src()) + flow->core.src_port = htons(atoi(value)); + break; + case POST_NAPT_DST_TRANSPORT_PORT: + if(ntop->getPrefs()->do_override_dst_with_post_nat_dst()) + flow->core.dst_port = htons(atoi(value)); + break; + case INGRESS_VRFID: + flow->core.vrfId = atoi(value); + break; + case IPV4_SRC_MASK: + case IPV4_DST_MASK: + if(strcmp(value, "0")) + return false; + break; + case IPV4_NEXT_HOP: + if(strcmp(value, "0.0.0.0")) + return false; + break; + + default: + return false; + } + + return true; +} + +/* **************************************************** */ + +bool ZMQParserInterface::parsePENNtopField(Parsed_Flow * const flow, u_int32_t field, const char * const value) const { + switch(field) { + case L7_PROTO: + if(!strchr(value, '.')) { + /* Old behaviour, only the app protocol */ + flow->core.l7_proto.app_protocol = atoi(value); + } else { + char *proto_dot; + + flow->core.l7_proto.master_protocol = (u_int16_t)strtoll(value, &proto_dot, 10); + flow->core.l7_proto.app_protocol = (u_int16_t)strtoll(proto_dot + 1, NULL, 10); + } + +#if 0 + ntop->getTrace()->traceEvent(TRACE_NORMAL, "[value: %s][master: %u][app: %u]", + value, + flow->core.l7_proto.master_protocol, + flow->core.l7_proto.app_protocol); +#endif + break; + case OOORDER_IN_PKTS: + flow->core.tcp.ooo_in_pkts = atol(value); + break; + case OOORDER_OUT_PKTS: + flow->core.tcp.ooo_out_pkts = atol(value); + break; + case RETRANSMITTED_IN_PKTS: + flow->core.tcp.retr_in_pkts = atol(value); + break; + case RETRANSMITTED_OUT_PKTS: + flow->core.tcp.retr_out_pkts = atol(value); + break; + /* TODO add lost in/out to nProbe and here */ + case CLIENT_NW_LATENCY_MS: + { + float client_nw_latency = atof(value); + flow->core.tcp.clientNwLatency.tv_sec = client_nw_latency / 1e3; + flow->core.tcp.clientNwLatency.tv_usec = 1e3 * (client_nw_latency - flow->core.tcp.clientNwLatency.tv_sec * 1e3); + break; + } + case SERVER_NW_LATENCY_MS: + { + float server_nw_latency = atof(value); + flow->core.tcp.serverNwLatency.tv_sec = server_nw_latency / 1e3; + flow->core.tcp.serverNwLatency.tv_usec = 1e3 * (server_nw_latency - flow->core.tcp.serverNwLatency.tv_sec * 1e3); + break; + } + case CLIENT_TCP_FLAGS: + flow->core.tcp.client_tcp_flags = atoi(value); + case SERVER_TCP_FLAGS: + flow->core.tcp.server_tcp_flags = atoi(value); + case APPL_LATENCY_MS: + flow->core.tcp.applLatencyMsec = atof(value); + break; + case DNS_QUERY: + flow->dns_query = strdup(value); + break; + case HTTP_URL: + flow->http_url = strdup(value); + break; + case HTTP_SITE: + flow->http_site = strdup(value); + break; + case SSL_SERVER_NAME: + flow->ssl_server_name = strdup(value); + break; + case BITTORRENT_HASH: + flow->bittorrent_hash = strdup(value); + break; + case NPROBE_IPV4_ADDRESS: + /* Do not override EXPORTER_IPV4_ADDRESS */ + if(flow->core.deviceIP == 0 && (flow->core.deviceIP = ntohl(inet_addr(value)))) + return false; + break; + default: + return false; + } + + return true; +} + +/* **************************************************** */ + +void ZMQParserInterface::parseSingleFlow(json_object *o, + u_int8_t source_id, + NetworkInterface *iface) { + Parsed_Flow flow; + IpAddress ip_aux; /* used to check empty IPs */ + struct json_object_iterator it = json_object_iter_begin(o); + struct json_object_iterator itEnd = json_object_iter_end(o); + bool invalid_flow = false; + + /* Reset data */ + memset(&flow, 0, sizeof(flow)); + flow.core.l7_proto.master_protocol = flow.core.l7_proto.app_protocol = NDPI_PROTOCOL_UNKNOWN; + flow.core.l7_proto.category = NDPI_PROTOCOL_CATEGORY_UNSPECIFIED; + flow.additional_fields = json_object_new_object(); + flow.core.pkt_sampling_rate = 1; /* 1:1 (no sampling) */ + flow.core.source_id = source_id, flow.core.vlan_id = 0; + + while(!json_object_iter_equal(&it, &itEnd)) { + const char *key = json_object_iter_peek_name(&it); + json_object *v = json_object_iter_peek_value(&it); + const char *value = json_object_get_string(v); + bool add_to_additional_fields = false; + + if((key != NULL) && (value != NULL)) { + u_int32_t pen, key_id; + json_object *additional_o = json_tokener_parse(value); + bool res; + + getKeyId((char*)key, &pen, &key_id); + + switch(pen) { + case 0: /* No PEN */ + res = parsePENZeroField(&flow, key_id, value); + if(res) + break; + /* Dont'break when res == false for backward compatibility: attempt to parse Zero-PEN as Ntop-PEN */ + case NTOP_PEN: + res = parsePENNtopField(&flow, key_id, value); + break; + case UNKNOWN_PEN: + default: + res = false; + break; + } + + if(!res) { + switch(key_id) { + case 0: //json additional object added by Flow::serialize() + if((additional_o != NULL) && (strcmp(key,"json") == 0)) { + struct json_object_iterator additional_it = json_object_iter_begin(additional_o); + struct json_object_iterator additional_itEnd = json_object_iter_end(additional_o); + + while(!json_object_iter_equal(&additional_it, &additional_itEnd)) { + + const char *additional_key = json_object_iter_peek_name(&additional_it); + json_object *additional_v = json_object_iter_peek_value(&additional_it); + const char *additional_value = json_object_get_string(additional_v); + + if((additional_key != NULL) && (additional_value != NULL)) { + json_object_object_add(flow.additional_fields, additional_key, + json_object_new_string(additional_value)); + } + json_object_iter_next(&additional_it); + } + } + break; + default: +#ifdef NTOPNG_PRO + if(custom_app_maps || (custom_app_maps = new(std::nothrow) CustomAppMaps())) + custom_app_maps->checkCustomApp(key, value, &flow); +#endif + ntop->getTrace()->traceEvent(TRACE_DEBUG, "Not handled ZMQ field %u/%s", key_id, key); + add_to_additional_fields = true; + break; + } /* switch */ + } + + if(add_to_additional_fields) + json_object_object_add(flow.additional_fields, + key, json_object_new_string(value)); + + if(additional_o) json_object_put(additional_o); + } /* if */ + + /* Move to the next element */ + json_object_iter_next(&it); + } // while json_object_iter_equal + + if(flow.core.vlan_id && ntop->getPrefs()->do_ignore_vlans()) + flow.core.vlan_id = 0; + + /* Handle zero IPv4/IPv6 discrepacies */ + if(flow.core.version == 0) { + if(flow.core.src_ip.getVersion() != flow.core.dst_ip.getVersion()) { + if(flow.core.dst_ip.isIPv4() && flow.core.src_ip.isIPv6() && flow.core.src_ip.isEmpty()) + flow.core.src_ip.setVersion(4); + else if(flow.core.src_ip.isIPv4() && flow.core.dst_ip.isIPv6() && flow.core.dst_ip.isEmpty()) + flow.core.dst_ip.setVersion(4); + else if(flow.core.dst_ip.isIPv6() && flow.core.src_ip.isIPv4() && flow.core.src_ip.isEmpty()) + flow.core.src_ip.setVersion(6); + else if(flow.core.src_ip.isIPv6() && flow.core.dst_ip.isIPv4() && flow.core.dst_ip.isEmpty()) + flow.core.dst_ip.setVersion(6); + else { + invalid_flow = true; + ntop->getTrace()->traceEvent(TRACE_WARNING, + "IP version mismatch: client:%d server:%d - flow will be ignored", + flow.core.src_ip.getVersion(), flow.core.dst_ip.getVersion()); + } + } + } else + flow.core.src_ip.setVersion(flow.core.version), flow.core.dst_ip.setVersion(flow.core.version); + + if(!invalid_flow) { + /* Process Flow */ + iface->processFlow(&flow); + } + + /* Dispose memory */ + if(flow.dns_query) free(flow.dns_query); + if(flow.http_url) free(flow.http_url); + if(flow.http_site) free(flow.http_site); + if(flow.ssl_server_name) free(flow.ssl_server_name); + if(flow.bittorrent_hash) free(flow.bittorrent_hash); + + // json_object_put(o); + json_object_put(flow.additional_fields); +} + +/* **************************************************** */ + +u_int8_t ZMQParserInterface::parseFlow(const char * const payload, int payload_size, u_int8_t source_id, void *data) { + json_object *f; + enum json_tokener_error jerr = json_tokener_success; + NetworkInterface *iface = (NetworkInterface*)data; + + // payload[payload_size] = '\0'; + // ntop->getTrace()->traceEvent(TRACE_NORMAL, "%s", payload); + + f = json_tokener_parse_verbose(payload, &jerr); + + if(f != NULL) { + int rc; + + if(json_object_get_type(f) == json_type_array) { + /* Flow array */ + int id, num_elements = json_object_array_length(f); + + for(id = 0; id < num_elements; id++) + parseSingleFlow(json_object_array_get_idx(f, id), source_id, iface); + + rc = num_elements; + } else { + parseSingleFlow(f, source_id, iface); + rc = 1; + } + + json_object_put(f); + return(rc); + } else { + // if o != NULL + if(!once) { + ntop->getTrace()->traceEvent(TRACE_WARNING, + "Invalid message received: your nProbe sender is outdated, data encrypted or invalid JSON?"); + ntop->getTrace()->traceEvent(TRACE_WARNING, "JSON Parse error [%s] payload size: %u payload: %s", + json_tokener_error_desc(jerr), + payload_size, + payload); + } + + once = true; + return 0; + } + + return 0; +} + +/* **************************************************** */ + +u_int8_t ZMQParserInterface::parseCounter(const char * const payload, int payload_size, u_int8_t source_id, void *data) { + json_object *o; + enum json_tokener_error jerr = json_tokener_success; + NetworkInterface * iface = (NetworkInterface*)data; + sFlowInterfaceStats stats; + + // payload[payload_size] = '\0'; + + memset(&stats, 0, sizeof(stats)); + o = json_tokener_parse_verbose(payload, &jerr); + + if(o != NULL) { + struct json_object_iterator it = json_object_iter_begin(o); + struct json_object_iterator itEnd = json_object_iter_end(o); + + /* Reset data */ + memset(&stats, 0, sizeof(stats)); + + while(!json_object_iter_equal(&it, &itEnd)) { + const char *key = json_object_iter_peek_name(&it); + json_object *v = json_object_iter_peek_value(&it); + const char *value = json_object_get_string(v); + + if((key != NULL) && (value != NULL)) { + if(!strcmp(key, "deviceIP")) stats.deviceIP = ntohl(inet_addr(value)); + else if(!strcmp(key, "ifIndex")) stats.ifIndex = atol(value); + else if(!strcmp(key, "ifType")) stats.ifType = atol(value); + else if(!strcmp(key, "ifSpeed")) stats.ifSpeed = atol(value); + else if(!strcmp(key, "ifDirection")) stats.ifFullDuplex = (!strcmp(value, "Full")) ? true : false; + else if(!strcmp(key, "ifAdminStatus")) stats.ifAdminStatus = (!strcmp(value, "Up")) ? true : false; + else if(!strcmp(key, "ifOperStatus")) stats.ifOperStatus = (!strcmp(value, "Up")) ? true : false; + else if(!strcmp(key, "ifInOctets")) stats.ifInOctets = atoll(value); + else if(!strcmp(key, "ifInPackets")) stats.ifInPackets = atoll(value); + else if(!strcmp(key, "ifInErrors")) stats.ifInErrors = atoll(value); + else if(!strcmp(key, "ifOutOctets")) stats.ifOutOctets = atoll(value); + else if(!strcmp(key, "ifOutPackets")) stats.ifOutPackets = atoll(value); + else if(!strcmp(key, "ifOutErrors")) stats.ifOutErrors = atoll(value); + else if(!strcmp(key, "ifPromiscuousMode")) stats.ifPromiscuousMode = (!strcmp(value, "1")) ? true : false; + } /* if */ + + /* Move to the next element */ + json_object_iter_next(&it); + } // while json_object_iter_equal + + /* Process Flow */ + iface->processInterfaceStats(&stats); + + json_object_put(o); + } else { + // if o != NULL + if(!once) +{ ntop->getTrace()->traceEvent(TRACE_WARNING, + "Invalid message received: your nProbe sender is outdated, data encrypted or invalid JSON?"); + ntop->getTrace()->traceEvent(TRACE_WARNING, "JSON Parse error [%s] payload size: %u payload: %s", + json_tokener_error_desc(jerr), + payload_size, + payload); + } + once = true; + return -1; + } + + return 0; +} + +/* **************************************************** */ + +u_int8_t ZMQParserInterface::parseTemplate(const char * const payload, int payload_size, u_int8_t source_id, void *data) { + /* The format that is currently defined for templates is a JSON as follows: + + [{"PEN":0,"field":1,"len":4,"format":"formatted_uint","name":"IN_BYTES","descr":"Incoming flow bytes (src->dst)"},{"PEN":0,"field":2,"len":4,"format":"formatted_uint","name":"IN_PKTS","descr":"Incoming flow packets (src->dst)"},] + */ + + // ntop->getTrace()->traceEvent(TRACE_NORMAL, "%s", payload); + + ZMQ_Template zmq_template; + json_object *obj, *w, *z; + enum json_tokener_error jerr = json_tokener_success; + + memset(&zmq_template, 0, sizeof(zmq_template)); + obj = json_tokener_parse_verbose(payload, &jerr); + + if(obj) { + if(json_object_get_type(obj) == json_type_array) { + int i, num_elements = json_object_array_length(obj); + + for(i = 0; i < num_elements; i++) { + memset(&zmq_template, 0, sizeof(zmq_template)); + + w = json_object_array_get_idx(obj, i); + + if(json_object_object_get_ex(w, "PEN", &z)) + zmq_template.pen = (u_int32_t)json_object_get_int(z); + + if(json_object_object_get_ex(w, "field", &z)) + zmq_template.field = (u_int32_t)json_object_get_int(z); + + if(json_object_object_get_ex(w, "format", &z)) + zmq_template.format = json_object_get_string(z); + + if(json_object_object_get_ex(w, "name", &z)) + zmq_template.name = json_object_get_string(z); + + if(json_object_object_get_ex(w, "descr", &z)) + zmq_template.descr = json_object_get_string(z); + + if(zmq_template.name) + addMapping(zmq_template.name, zmq_template.field, zmq_template.pen); + + // ntop->getTrace()->traceEvent(TRACE_NORMAL, "Template [PEN: %u][field: %u][format: %s][name: %s][descr: %s]", + // zmq_template.pen, zmq_template.field, zmq_template.format, zmq_template.name, zmq_template.descr) + ; + } + } + json_object_put(obj); + } else { + // if o != NULL + if(!once) { + ntop->getTrace()->traceEvent(TRACE_WARNING, + "Invalid message received: your nProbe sender is outdated, data encrypted or invalid JSON?"); + ntop->getTrace()->traceEvent(TRACE_WARNING, "JSON Parse error [%s] payload size: %u payload: %s", + json_tokener_error_desc(jerr), + payload_size, + payload); + } + once = true; + return -1; + } + + return 0; +} + +/* **************************************************** */ + +void ZMQParserInterface::setFieldMap(const ZMQ_FieldMap * const field_map) const { + char hname[CONST_MAX_LEN_REDIS_KEY], key[32]; + snprintf(hname, sizeof(hname), CONST_FIELD_MAP_CACHE_KEY, get_id(), field_map->pen); + snprintf(key, sizeof(key), "%u", field_map->field); + + ntop->getRedis()->hashSet(hname, key, field_map->map); +} + +/* **************************************************** */ + +void ZMQParserInterface::setFieldValueMap(const ZMQ_FieldValueMap * const field_value_map) const { + char hname[CONST_MAX_LEN_REDIS_KEY], key[32]; + snprintf(hname, sizeof(hname), CONST_FIELD_VALUE_MAP_CACHE_KEY, get_id(), field_value_map->pen, field_value_map->field); + snprintf(key, sizeof(key), "%u", field_value_map->value); + + ntop->getRedis()->hashSet(hname, key, field_value_map->map); +} + +/* **************************************************** */ + +u_int8_t ZMQParserInterface::parseOptionFieldMap(json_object * const jo) const { + int arraylen = json_object_array_length(jo); + json_object *w, *z; + ZMQ_FieldMap field_map; + memset(&field_map, 0, sizeof(field_map)); + + for(int i = 0; i < arraylen; i++) { + w = json_object_array_get_idx(jo, i); + + if(json_object_object_get_ex(w, "PEN", &z)) + field_map.pen = (u_int32_t)json_object_get_int(z); + + if(json_object_object_get_ex(w, "field", &z)) { + field_map.field = (u_int32_t)json_object_get_int(z); + + if(json_object_object_get_ex(w, "map", &z)) { + field_map.map = json_object_to_json_string(z); + + setFieldMap(&field_map); + +#ifdef CUSTOM_APP_DEBUG + ntop->getTrace()->traceEvent(TRACE_NORMAL, "Option FieldMap [PEN: %u][field: %u][map: %s]", + field_map.pen, field_map.field, field_map.map); +#endif + } + } + } + + return 0; +} + +/* **************************************************** */ + +u_int8_t ZMQParserInterface::parseOptionFieldValueMap(json_object * const w) const { + json_object *z; + ZMQ_FieldValueMap field_value_map; + memset(&field_value_map, 0, sizeof(field_value_map)); + + if(json_object_object_get_ex(w, "PEN", &z)) + field_value_map.pen = (u_int32_t)json_object_get_int(z); + + if(json_object_object_get_ex(w, "field", &z)) { + field_value_map.field = (u_int32_t)json_object_get_int(z); + + if(json_object_object_get_ex(w, "value", &z)) { + field_value_map.value = (u_int32_t)json_object_get_int(z); + + if(json_object_object_get_ex(w, "map", &z)) { + field_value_map.map = json_object_to_json_string(z); + + setFieldValueMap(&field_value_map); + +#ifdef CUSTOM_APP_DEBUG + ntop->getTrace()->traceEvent(TRACE_NORMAL, "Option FieldValueMap [PEN: %u][field: %u][value: %u][map: %s]", + field_value_map.pen, field_value_map.field, field_value_map.value, field_value_map.map); +#endif + } + } + } + + return 0; +} + +/* **************************************************** */ + +u_int8_t ZMQParserInterface::parseOption(const char * const payload, int payload_size, u_int8_t source_id, void *data) { + /* The format that is currently defined for options is a JSON as follows: + + char opt[] = " + "{\"PEN\":8741, \"field\": 22, \"value\":1, \"map\":{\"name\":\"Skype\"}}," + "{\"PEN\":8741, \"field\": 22, \"value\":3, \"map\":{\"name\":\"Winni\"}}"; + + parseOption(opt, strlen(opt), source_id, this); + */ + + // ntop->getTrace()->traceEvent(TRACE_NORMAL, "%s", payload); + + json_object *o; + enum json_tokener_error jerr = json_tokener_success; + + o = json_tokener_parse_verbose(payload, &jerr); + + if(o != NULL) { + parseOptionFieldValueMap(o); + json_object_put(o); + } else { + // if o != NULL + if(!once) { + ntop->getTrace()->traceEvent(TRACE_WARNING, + "Invalid message received: your nProbe sender is outdated, data encrypted or invalid JSON?"); + ntop->getTrace()->traceEvent(TRACE_WARNING, "JSON Parse error [%s] payload size: %u payload: %s", + json_tokener_error_desc(jerr), + payload_size, + payload); + } + once = true; + return -1; + } + + return 0; +} + +/* **************************************** */ + +void ZMQParserInterface::setRemoteStats(ZMQ_RemoteStats *zrs) { + if(!zrs) return; + + ifSpeed = zrs->remote_ifspeed, last_pkt_rcvd = 0, last_pkt_rcvd_remote = zrs->remote_time, + last_remote_pps = zrs->avg_pps, last_remote_bps = zrs->avg_bps; + + if((zmq_initial_pkts == 0) /* ntopng has been restarted */ + || (zrs->remote_bytes < zmq_initial_bytes) /* nProbe has been restarted */ + ) { + /* Start over */ + zmq_initial_bytes = zrs->remote_bytes, zmq_initial_pkts = zrs->remote_pkts; + } + + if(zmq_remote_initial_exported_flows == 0 /* ntopng has been restarted */ + || zrs->num_flow_exports < zmq_remote_initial_exported_flows) /* nProbe has been restarted */ + zmq_remote_initial_exported_flows = zrs->num_flow_exports; + + if(zmq_remote_stats_shadow) free(zmq_remote_stats_shadow); + zmq_remote_stats_shadow = zmq_remote_stats; + zmq_remote_stats = zrs; + + /* + * Don't override ethStats here, these stats are properly updated + * inside NetworkInterface::processFlow for ZMQ interfaces. + * Overriding values here may cause glitches and non-strictly-increasing counters + * yielding negative rates. + ethStats.setNumBytes(zrs->remote_bytes), ethStats.setNumPackets(zrs->remote_pkts); + * + */ +} + +/* **************************************************** */ + +#ifdef NTOPNG_PRO +bool ZMQParserInterface::getCustomAppDetails(u_int32_t remapped_app_id, u_int32_t *const pen, u_int32_t *const app_field, u_int32_t *const app_id) { + return custom_app_maps && custom_app_maps->getCustomAppDetails(remapped_app_id, pen, app_field, app_id); +} +#endif + +/* **************************************************** */ + +void ZMQParserInterface::lua(lua_State* vm) { + ZMQ_RemoteStats *zrs = zmq_remote_stats; + + NetworkInterface::lua(vm); + + if(zrs) { + if(zrs->remote_ifname[0] != '\0') + lua_push_str_table_entry(vm, "remote.name", zrs->remote_ifname); + if(zrs->remote_ifaddress[0] != '\0') + lua_push_str_table_entry(vm, "remote.if_addr",zrs->remote_ifaddress); + if(zrs->remote_probe_address[0] != '\0') + lua_push_str_table_entry(vm, "probe.ip", zrs->remote_probe_address); + if(zrs->remote_probe_public_address[0] != '\0') + lua_push_str_table_entry(vm, "probe.public_ip", zrs->remote_probe_public_address); + + lua_push_uint64_table_entry(vm, "zmq.num_flow_exports", zrs->num_flow_exports - zmq_remote_initial_exported_flows); + lua_push_uint64_table_entry(vm, "zmq.num_exporters", zrs->num_exporters); + + if(zrs->export_queue_full > 0) + lua_push_uint64_table_entry(vm, "zmq.drops.export_queue_full", zrs->export_queue_full); + lua_push_uint64_table_entry(vm, "zmq.drops.flow_collection_drops", zrs->flow_collection_drops); + + lua_push_uint64_table_entry(vm, "timeout.lifetime", zrs->remote_lifetime_timeout); + lua_push_uint64_table_entry(vm, "timeout.idle", zrs->remote_idle_timeout); + } +} + +/* **************************************************** */ + +#endif diff --git a/src/main.cpp b/src/main.cpp index d736ffab3bc0..ce39a354463a 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -187,7 +187,9 @@ int main(int argc, char *argv[]) else endpoint = ifName; - iface = new CollectorInterface(endpoint); + iface = new ZMQCollectorInterface(endpoint); + } else if(strstr(ifName, "syslog://")) { + iface = new SyslogCollectorInterface(ifName); #if defined(HAVE_PF_RING) && (!defined(NTOPNG_EMBEDDED_EDITION)) && (!defined(__i686__)) && (!defined(__ARM_ARCH)) } else if(strstr(ifName, "zcflow:")) { iface = new ZCCollectorInterface(ifName);