Skip to content

Commit

Permalink
Merge pull request #11 from ntop/dev
Browse files Browse the repository at this point in the history
sync
  • Loading branch information
SalvatoreCostantino authored Apr 2, 2019
2 parents f002b94 + 6040e76 commit bece7b6
Show file tree
Hide file tree
Showing 39 changed files with 2,043 additions and 1,263 deletions.
4 changes: 3 additions & 1 deletion include/ArpStatsHashMatrix.h
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,9 @@ class ArpStatsHashMatrix : public GenericHash {
public:
ArpStatsHashMatrix(NetworkInterface *iface, u_int _num_hashes, u_int _max_hash_size);

ArpStatsMatrixElement* get(const u_int8_t _src_mac[6], const u_int8_t _dst_mac[6], bool * const src2dst);
ArpStatsMatrixElement* get(const u_int8_t _src_mac[6],
const u_int32_t _src_ip, const u_int32_t _dst_ip,
bool * const src2dst);
void lua(lua_State* vm);
};

Expand Down
15 changes: 10 additions & 5 deletions include/ArpStatsMatrixElement.h
Original file line number Diff line number Diff line change
Expand Up @@ -33,27 +33,32 @@ class ArpStatsMatrixElement : public GenericHashEntry {
} src2dst, dst2src;
} stats;

u_int8_t src_mac[6];
u_int8_t dst_mac[6];
u_int8_t src_mac[6], dst_mac[6];
u_int32_t src_ip, dst_ip;

public:
ArpStatsMatrixElement(NetworkInterface *_iface, const u_int8_t _src_mac[6], const u_int8_t _dst_mac[6], bool * const src2dst);
ArpStatsMatrixElement(NetworkInterface *_iface,
const u_int8_t _src_mac[6], const u_int8_t _dst_mac[6],
const u_int32_t _src_ip, const u_int32_t _dst_ip);
~ArpStatsMatrixElement();

inline void incArpReplies(bool src2dst) {
src2dst ? stats.src2dst.replies++ : stats.dst2src.replies++;
updateSeen();
}

inline void incArpRequests(bool src2dst) {
src2dst ? stats.src2dst.requests++ : stats.dst2src.requests++;
updateSeen();
}

bool equal(const u_int8_t _src_mac[6], const u_int8_t _dst_mac[6], bool * const src2dst) const;
bool equal(const u_int8_t _src_mac[6],
const u_int32_t _src_ip, const u_int32_t _dst_ip,
bool * const src2dst) const;
virtual bool idle();
u_int32_t key();
void lua(lua_State* vm);
void print() const;
void print(char *msg) const;
};

#endif /* _ARP_STATS_MATRIX_ELEMENT_H_ */
Expand Down
2 changes: 1 addition & 1 deletion include/DummyInterface.h
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@

#ifndef HAVE_NEDGE

class DummyInterface : public ParserInterface {
class DummyInterface : public ZMQParserInterface {
private:
inline u_int32_t getNumDroppedPackets() { return(0); };

Expand Down
40 changes: 17 additions & 23 deletions include/Flow.h
Original file line number Diff line number Diff line change
Expand Up @@ -184,18 +184,14 @@ class Flow : public GenericHashEntry {
void updatePacketStats(InterarrivalStats *stats, const struct timeval *when);
void dumpPacketStats(lua_State* vm, bool cli2srv_direction);
bool isReadyToPurge();
inline bool isBlacklistedFlow() {
return(cli_host && srv_host && (cli_host->isBlacklisted()
|| srv_host->isBlacklisted()
|| (get_protocol_category() == CUSTOM_CATEGORY_MALWARE)));
};
bool isBlacklistedFlow() const;
inline bool isDeviceAllowedProtocol() {
return(!cli_host || !srv_host ||
((cli_host->getDeviceAllowedProtocolStatus(ndpiDetectedProtocol, true) == device_proto_allowed) &&
(srv_host->getDeviceAllowedProtocolStatus(ndpiDetectedProtocol, false) == device_proto_allowed)));
}
char* printTCPflags(u_int8_t flags, char * const buf, u_int buf_len) const;
inline bool isProto(u_int16_t p ) { return((ndpi_get_lower_proto(ndpiDetectedProtocol) == p) ? true : false); }
inline bool isProto(u_int16_t p ) const { return((ndpi_get_lower_proto(ndpiDetectedProtocol) == p) ? true : false); }
#ifdef NTOPNG_PRO
void update_pools_stats(const struct timeval *tv,
u_int64_t diff_sent_packets, u_int64_t diff_sent_bytes,
Expand All @@ -222,12 +218,12 @@ class Flow : public GenericHashEntry {
struct site_categories* getFlowCategory(bool force_categorization);
void freeDPIMemory();
bool isTiny();
inline bool isSSL() { return(isProto(NDPI_PROTOCOL_SSL)); }
inline bool isSSH() { return(isProto(NDPI_PROTOCOL_SSH)); }
inline bool isDNS() { return(isProto(NDPI_PROTOCOL_DNS)); }
inline bool isDHCP() { return(isProto(NDPI_PROTOCOL_DHCP)); }
inline bool isHTTP() { return(isProto(NDPI_PROTOCOL_HTTP)); }
inline bool isICMP() { return(isProto(NDPI_PROTOCOL_IP_ICMP) || isProto(NDPI_PROTOCOL_IP_ICMPV6)); }
inline bool isSSL() const { return(isProto(NDPI_PROTOCOL_SSL)); }
inline bool isSSH() const { return(isProto(NDPI_PROTOCOL_SSH)); }
inline bool isDNS() const { return(isProto(NDPI_PROTOCOL_DNS)); }
inline bool isDHCP() const { return(isProto(NDPI_PROTOCOL_DHCP)); }
inline bool isHTTP() const { return(isProto(NDPI_PROTOCOL_HTTP)); }
inline bool isICMP() const { return(isProto(NDPI_PROTOCOL_IP_ICMP) || isProto(NDPI_PROTOCOL_IP_ICMPV6)); }
inline bool isMaskedFlow() {
return(!get_cli_host() || Utils::maskHost(get_cli_host()->isLocalHost())
|| !get_srv_host() || Utils::maskHost(get_srv_host()->isLocalHost()));
Expand Down Expand Up @@ -298,7 +294,7 @@ class Flow : public GenericHashEntry {
void addFlowStats(bool cli2srv_direction, u_int in_pkts, u_int in_bytes, u_int in_goodput_bytes,
u_int out_pkts, u_int out_bytes, u_int out_goodput_bytes, time_t last_seen);
inline bool isThreeWayHandshakeOK() { return(twh_ok); };
inline bool isDetectionCompleted() { return(detection_completed); };
inline bool isDetectionCompleted() const { return(detection_completed); };
inline struct ndpi_flow_struct* get_ndpi_flow() { return(ndpiFlow); };
inline void* get_cli_id() { return(cli_id); };
inline void* get_srv_id() { return(srv_id); };
Expand Down Expand Up @@ -330,26 +326,26 @@ class Flow : public GenericHashEntry {
inline time_t get_partial_first_seen() { return(last_db_dump.last_dump == 0 ? get_first_seen() : last_db_dump.last_dump); };
inline time_t get_partial_last_seen() { return(get_last_seen()); };
inline u_int32_t get_duration() { return((u_int32_t)(get_last_seen()-get_first_seen())); };
inline char* get_protocol_name() { return(Utils::l4proto2name(protocol)); };
inline ndpi_protocol get_detected_protocol() { return(isDetectionCompleted() ? ndpiDetectedProtocol : ndpiUnknownProtocol); };
inline char* get_protocol_name() const { return(Utils::l4proto2name(protocol)); };
inline ndpi_protocol get_detected_protocol() const { return(isDetectionCompleted() ? ndpiDetectedProtocol : ndpiUnknownProtocol); };

inline Host* get_cli_host() { return(cli_host); };
inline Host* get_srv_host() { return(srv_host); };
inline char* get_json_info() { return(json_info); };
inline ndpi_protocol_breed_t get_protocol_breed() {
inline ndpi_protocol_breed_t get_protocol_breed() const {
return(ndpi_get_proto_breed(iface->get_ndpi_struct(), isDetectionCompleted() ? ndpiDetectedProtocol.app_protocol : NDPI_PROTOCOL_UNKNOWN));
};
inline const char * const get_protocol_breed_name() {
inline const char * const get_protocol_breed_name() const {
return(ndpi_get_proto_breed_name(iface->get_ndpi_struct(), get_protocol_breed()));
};
inline ndpi_protocol_category_t get_protocol_category() {
inline ndpi_protocol_category_t get_protocol_category() const {
return(ndpi_get_proto_category(iface->get_ndpi_struct(),
isDetectionCompleted() ? ndpiDetectedProtocol : ndpiUnknownProtocol));
};
inline const char * const get_protocol_category_name() {
inline const char * const get_protocol_category_name() const {
return(ndpi_category_get_name(iface->get_ndpi_struct(), get_protocol_category()));
};
char* get_detected_protocol_name(char *buf, u_int buf_len) {
char* get_detected_protocol_name(char *buf, u_int buf_len) const {
return(ndpi_protocol2name(iface->get_ndpi_struct(),
isDetectionCompleted() ? ndpiDetectedProtocol : ndpiUnknownProtocol,
buf, buf_len));
Expand All @@ -366,8 +362,7 @@ class Flow : public GenericHashEntry {
u_int64_t get_current_packets_cli2srv();
u_int64_t get_current_packets_srv2cli();
inline bool idle() { return(is_ready_to_be_purged()); }
inline bool is_l7_protocol_guessed() { return(l7_protocol_guessed); };
char* print(char *buf, u_int buf_len);
char* print(char *buf, u_int buf_len) const;
void update_hosts_stats(struct timeval *tv, bool dump_alert);
u_int32_t key();
static u_int32_t key(Host *cli, u_int16_t cli_port,
Expand All @@ -382,7 +377,6 @@ class Flow : public GenericHashEntry {
bool *src2srv_direction);
bool clientLessThanServer() const;
void sumStats(nDPIStats *stats);
void guessProtocol();
bool dumpFlow(bool dump_alert);
bool match(AddressTree *ptree);
void dissectHTTP(bool src2dst_direction, char *payload, u_int16_t payload_len);
Expand Down
1 change: 1 addition & 0 deletions include/Host.h
Original file line number Diff line number Diff line change
Expand Up @@ -238,6 +238,7 @@ class Host : public GenericHashEntry {
virtual void incNumDNSResponsesSent(u_int32_t ret_code) { };
virtual void incNumDNSResponsesRcvd(u_int32_t ret_code) { };
virtual void luaDNS(lua_State *vm) const { };
virtual void luaTCP(lua_State *vm) const { };
virtual void postHashAdd();

virtual NetworkStats* getNetworkStats(int16_t networkId) { return(NULL); };
Expand Down
2 changes: 1 addition & 1 deletion include/HostStats.h
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ class HostStats: public Checkpointable, public TimeseriesStats {
virtual void decNumFlows(bool as_client, Host *peer) {};
virtual bool hasAnomalies(time_t when) { return false; };
virtual void luaAnomalies(lua_State* vm, time_t when) {};
virtual void lua(lua_State* vm, bool mask_host, bool host_details, bool verbose, bool tsLua = false);
virtual void lua(lua_State* vm, bool mask_host, DetailsLevel details_level, bool tsLua = false);

#ifdef NTOPNG_PRO
inline void incQuotaEnforcementStats(time_t when, u_int16_t ndpi_proto,
Expand Down
1 change: 1 addition & 0 deletions include/LocalHost.h
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ class LocalHost : public Host {
virtual void luaDNS(lua_State *vm) const { stats->luaDNS(vm,false); };
virtual void incrVisitedWebSite(char *hostname) { stats->incrVisitedWebSite(hostname); };
virtual HTTPstats* getHTTPstats() { return(stats->getHTTPstats()); };
virtual void luaTCP(lua_State *vm) const { stats->lua(vm,false,details_normal); };

virtual void lua(lua_State* vm, AddressTree * ptree, bool host_details,
bool verbose, bool returnHost, bool asListElement);
Expand Down
2 changes: 1 addition & 1 deletion include/LocalHostStats.h
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ class LocalHostStats: public HostStats {
virtual void updateStats(struct timeval *tv);
virtual void getJSONObject(json_object *my_object, DetailsLevel details_level);
virtual void deserialize(json_object *obj);
virtual void lua(lua_State* vm, bool mask_host, bool host_details, bool verbose, bool tsLua = false);
virtual void lua(lua_State* vm, bool mask_host, DetailsLevel details_level, bool tsLua = false);
virtual void incNumFlows(bool as_client, Host *peer);
virtual void decNumFlows(bool as_client, Host *peer);

Expand Down
6 changes: 4 additions & 2 deletions include/NetworkInterface.h
Original file line number Diff line number Diff line change
Expand Up @@ -420,7 +420,7 @@ class NetworkInterface : public Checkpointable {
const u_char *packet,
u_int16_t *ndpiProtocol,
Host **srcHost, Host **dstHost, Flow **flow);
void processFlow(ZMQ_Flow *zflow);
void processFlow(Parsed_Flow *zflow);
void processInterfaceStats(sFlowInterfaceStats *stats);
void getnDPIStats(nDPIStats *stats, AddressTree *allowed_hosts, const char *host_ip, u_int16_t vlan_id);
void periodicStatsUpdate();
Expand Down Expand Up @@ -523,7 +523,9 @@ class NetworkInterface : public Checkpointable {

void runHousekeepingTasks();
void runShutdownTasks();
ArpStatsMatrixElement* getArpHashMatrixElement(const u_int8_t _src_mac[6], const u_int8_t _dst_mac[6], bool * const src2dst);
ArpStatsMatrixElement* getArpHashMatrixElement(const u_int8_t _src_mac[6], const u_int8_t _dst_mac[6],
const u_int32_t _src_ip, const u_int32_t _dst_ip,
bool * const src2dst);
Vlan* getVlan(u_int16_t vlanId, bool createIfNotPresent);
AutonomousSystem *getAS(IpAddress *ipa, bool createIfNotPresent);
Country* getCountry(const char *country_name, bool createIfNotPresent);
Expand Down
33 changes: 0 additions & 33 deletions include/ParserInterface.h
Original file line number Diff line number Diff line change
Expand Up @@ -26,43 +26,10 @@

class ParserInterface : public NetworkInterface {
private:
typedef std::pair<u_int32_t, u_int32_t> pen_value_t;
typedef std::map<string, pen_value_t > labels_map_t;
labels_map_t labels_map;
bool once;
u_int64_t zmq_initial_bytes, zmq_initial_pkts,
zmq_remote_initial_exported_flows;
ZMQ_RemoteStats *zmq_remote_stats, *zmq_remote_stats_shadow;
#ifdef NTOPNG_PRO
CustomAppMaps *custom_app_maps;
#endif
bool getKeyId(char *sym, u_int32_t * const pen, u_int32_t * const field) const;
void addMapping(const char *sym, u_int32_t num, u_int32_t pen = 0);
bool parsePENZeroField(ZMQ_Flow * const flow, u_int32_t field, const char * const value) const;
bool parsePENNtopField(ZMQ_Flow * const flow, u_int32_t field, const char * const value) const;
void parseSingleFlow(json_object *o, u_int8_t source_id, NetworkInterface *iface);
void setFieldMap(const ZMQ_FieldMap * const field_map) const;
void setFieldValueMap(const ZMQ_FieldValueMap * const field_value_map) const;

u_int8_t parseOptionFieldMap(json_object * const jo) const;
u_int8_t parseOptionFieldValueMap(json_object * const jo) const;

public:
ParserInterface(const char *endpoint, const char *custom_interface_type = NULL);
~ParserInterface();

u_int8_t parseFlow(const char * const payload, int payload_size, u_int8_t source_id, void *data);
u_int8_t parseEvent(const char * const payload, int payload_size, u_int8_t source_id, void *data);
u_int8_t parseCounter(const char * const payload, int payload_size, u_int8_t source_id, void *data);
u_int8_t parseTemplate(const char * const payload, int payload_size, u_int8_t source_id, void *data);
u_int8_t parseOption(const char * const payload, int payload_size, u_int8_t source_id, void *data);

virtual void setRemoteStats(ZMQ_RemoteStats *zrs);
#ifdef NTOPNG_PRO
virtual bool getCustomAppDetails(u_int32_t remapped_app_id, u_int32_t *const pen, u_int32_t *const app_field, u_int32_t *const app_id);
#endif
u_int32_t getNumDroppedPackets() { return zmq_remote_stats ? zmq_remote_stats->sflow_pkt_sample_drops : 0; };
virtual void lua(lua_State* vm);
};

#endif /* _PARSER_INTERFACE_H_ */
Expand Down
75 changes: 75 additions & 0 deletions include/SyslogCollectorInterface.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
/*
*
* (C) 2019 - ntop.org
*
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*
*/

#ifndef _SYSLOG_COLLECTOR_INTERFACE_H_
#define _SYSLOG_COLLECTOR_INTERFACE_H_

#include "ntop_includes.h"

#ifndef HAVE_NEDGE

class LuaEngine;

typedef struct {
int socket;
struct sockaddr_in address;
} syslog_client;

class SyslogCollectorInterface : public SyslogParserInterface {
private:
char *endpoint;
struct sockaddr_in listen_addr;
int listen_sock;
syslog_client connections[MAX_ZMQ_SUBSCRIBERS];

struct {
u_int32_t num_flows;
} recvStats;

public:
SyslogCollectorInterface(const char *_endpoint);
~SyslogCollectorInterface();

int initFDSets(fd_set *read_fds, fd_set *write_fds, fd_set *except_fds);
int handleNewConnection();
char *clientAddr2Str(syslog_client *client, char *buff);
void closeConnection(syslog_client *client);
int receiveFromClient(syslog_client *client);

inline const char* get_type() { return(CONST_INTERFACE_TYPE_SYSLOG); };
inline InterfaceType getIfType() { return(interface_type_SYSLOG); }
inline bool is_ndpi_enabled() { return(false); };
inline char* getEndpoint(u_int8_t id) { return(endpoint); };
inline bool isPacketInterface() { return(false); };
void collect_flows();

virtual void purgeIdle(time_t when);

void startPacketPolling();
void shutdown();
bool set_packet_filter(char *filter);
virtual void lua(lua_State* vm);
};

#endif /* HAVE_NEDGE */

#endif /* _SYSLOG_COLLECTOR_INTERFACE_H_ */

42 changes: 42 additions & 0 deletions include/SyslogParserInterface.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
/*
*
* (C) 2019 - ntop.org
*
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*
*/

#ifndef _SYSLOG_PARSER_INTERFACE_H_
#define _SYSLOG_PARSER_INTERFACE_H_

#include "ntop_includes.h"

class SyslogParserInterface : public ParserInterface {
private:

public:
SyslogParserInterface(const char *endpoint, const char *custom_interface_type = NULL);
~SyslogParserInterface();

u_int8_t parseLog(char *log_line, void *data);

u_int32_t getNumDroppedPackets() { return 0; };
virtual void lua(lua_State* vm);
};

#endif /* _SYSLOG_PARSER_INTERFACE_H_ */


2 changes: 2 additions & 0 deletions include/Utils.h
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ class Utils {
static char* formatTraffic(float numBits, bool bits, char *buf, u_int buf_len);
static char* formatPackets(float numPkts, char *buf, u_int buf_len);
static char* l4proto2name(u_int8_t proto);
static u_int8_t l4name2proto(char *name);
static bool isIPAddress(char *name);
static int setThreadAffinity(pthread_t thread, int core_id);
static void setThreadName(const char *name);
Expand Down Expand Up @@ -123,6 +124,7 @@ class Utils {
static bool mg_write_retry(struct mg_connection *conn, u_char *b, int len);
static bool parseAuthenticatorJson(HTTPAuthenticator *auth, char *content);
static void freeAuthenticator(HTTPAuthenticator *auth);
static DetailsLevel bool2DetailsLevel(bool max, bool higher,bool normal = false);

/* Patricia Tree */
static patricia_node_t* add_to_ptree(patricia_tree_t *tree, int family, void *addr, int bits);
Expand Down
Loading

0 comments on commit bece7b6

Please sign in to comment.