Implemented SyslogCollectorInterface/SyslogParserInterface

Implemented TCP-based syslog collector (multi producer)
Initial Suricata EVE events parsing (basic flows)
Reworked ZMQ CollectorInterface/Parser interface hierarchy

include/DummyInterface.h

@@ -26,7 +26,7 @@
 
 #ifndef HAVE_NEDGE
 
-class DummyInterface : public ParserInterface {
+class DummyInterface : public ZMQParserInterface {
  private:
   inline u_int32_t getNumDroppedPackets()   { return(0); };
 

include/NetworkInterface.h

@@ -420,7 +420,7 @@ class NetworkInterface : public Checkpointable {
 		     const u_char *packet,
 		     u_int16_t *ndpiProtocol,
 		     Host **srcHost, Host **dstHost, Flow **flow);
-  void processFlow(ZMQ_Flow *zflow);
+  void processFlow(Parsed_Flow *zflow);
   void processInterfaceStats(sFlowInterfaceStats *stats);
   void getnDPIStats(nDPIStats *stats, AddressTree *allowed_hosts, const char *host_ip, u_int16_t vlan_id);
   void periodicStatsUpdate();

include/ParserInterface.h

@@ -26,43 +26,10 @@
 
 class ParserInterface : public NetworkInterface {
  private:
-  typedef std::pair<u_int32_t, u_int32_t> pen_value_t;
-  typedef std::map<string, pen_value_t > labels_map_t;
-  labels_map_t labels_map;
-  bool once;
-  u_int64_t zmq_initial_bytes, zmq_initial_pkts,
-    zmq_remote_initial_exported_flows;
-  ZMQ_RemoteStats *zmq_remote_stats, *zmq_remote_stats_shadow;
-#ifdef NTOPNG_PRO
-  CustomAppMaps *custom_app_maps;
-#endif
-  bool getKeyId(char *sym, u_int32_t * const pen, u_int32_t * const field) const;
-  void addMapping(const char *sym, u_int32_t num, u_int32_t pen = 0);
-  bool parsePENZeroField(ZMQ_Flow * const flow, u_int32_t field, const char * const value) const;
-  bool parsePENNtopField(ZMQ_Flow * const flow, u_int32_t field, const char * const value) const;
-  void parseSingleFlow(json_object *o, u_int8_t source_id, NetworkInterface *iface);
-  void setFieldMap(const ZMQ_FieldMap * const field_map) const;
-  void setFieldValueMap(const ZMQ_FieldValueMap * const field_value_map) const;
-
-  u_int8_t parseOptionFieldMap(json_object * const jo) const;
-  u_int8_t parseOptionFieldValueMap(json_object * const jo) const;
 
  public:
   ParserInterface(const char *endpoint, const char *custom_interface_type = NULL);
   ~ParserInterface();
-
-  u_int8_t parseFlow(const char * const payload, int payload_size, u_int8_t source_id, void *data);
-  u_int8_t parseEvent(const char * const payload, int payload_size, u_int8_t source_id, void *data);
-  u_int8_t parseCounter(const char * const payload, int payload_size, u_int8_t source_id, void *data);
-  u_int8_t parseTemplate(const char * const payload, int payload_size, u_int8_t source_id, void *data);
-  u_int8_t parseOption(const char * const payload, int payload_size, u_int8_t source_id, void *data);
-
-  virtual void setRemoteStats(ZMQ_RemoteStats *zrs);
-#ifdef NTOPNG_PRO
-  virtual bool getCustomAppDetails(u_int32_t remapped_app_id, u_int32_t *const pen, u_int32_t *const app_field, u_int32_t *const app_id);
-#endif
-  u_int32_t getNumDroppedPackets() { return zmq_remote_stats ? zmq_remote_stats->sflow_pkt_sample_drops : 0; };
-  virtual void lua(lua_State* vm);
 };
 
 #endif /* _PARSER_INTERFACE_H_ */

include/SyslogCollectorInterface.h

@@ -0,0 +1,75 @@
+/*
+ *
+ * (C) 2019 - ntop.org
+ *
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ */
+
+#ifndef _SYSLOG_COLLECTOR_INTERFACE_H_
+#define _SYSLOG_COLLECTOR_INTERFACE_H_
+
+#include "ntop_includes.h"
+
+#ifndef HAVE_NEDGE
+
+class LuaEngine;
+
+typedef struct {
+  int socket;
+  struct sockaddr_in address;
+} syslog_client;
+
+class SyslogCollectorInterface : public SyslogParserInterface {
+ private:
+  char *endpoint;
+  struct sockaddr_in listen_addr;
+  int listen_sock;
+  syslog_client connections[MAX_ZMQ_SUBSCRIBERS];
+
+  struct {
+    u_int32_t num_flows;
+  } recvStats;
+
+ public:
+  SyslogCollectorInterface(const char *_endpoint);
+  ~SyslogCollectorInterface();
+
+  int initFDSets(fd_set *read_fds, fd_set *write_fds, fd_set *except_fds);
+  int handleNewConnection();
+  char *clientAddr2Str(syslog_client *client, char *buff);
+  void closeConnection(syslog_client *client);
+  int receiveFromClient(syslog_client *client);
+
+  inline const char* get_type()         { return(CONST_INTERFACE_TYPE_SYSLOG); };
+  inline InterfaceType getIfType()      { return(interface_type_SYSLOG); }
+  inline bool is_ndpi_enabled()         { return(false);      };
+  inline char* getEndpoint(u_int8_t id) { return(endpoint); };
+  inline bool isPacketInterface()       { return(false);      };
+  void collect_flows();
+
+  virtual void purgeIdle(time_t when);
+
+  void startPacketPolling();
+  void shutdown();
+  bool set_packet_filter(char *filter);
+  virtual void lua(lua_State* vm);
+};
+
+#endif /* HAVE_NEDGE */
+
+#endif /* _SYSLOG_COLLECTOR_INTERFACE_H_ */
+

include/SyslogParserInterface.h

@@ -0,0 +1,42 @@
+/*
+ *
+ * (C) 2019 - ntop.org
+ *
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ */
+
+#ifndef _SYSLOG_PARSER_INTERFACE_H_
+#define _SYSLOG_PARSER_INTERFACE_H_
+
+#include "ntop_includes.h"
+
+class SyslogParserInterface : public ParserInterface {
+ private:
+
+ public:
+  SyslogParserInterface(const char *endpoint, const char *custom_interface_type = NULL);
+  ~SyslogParserInterface();
+
+  u_int8_t parseLog(char *log_line, void *data);
+
+  u_int32_t getNumDroppedPackets() { return 0; };
+  virtual void lua(lua_State* vm);
+};
+
+#endif /* _SYSLOG_PARSER_INTERFACE_H_ */
+
+

include/Utils.h

@@ -42,6 +42,7 @@ class Utils {
   static char* formatTraffic(float numBits, bool bits, char *buf, u_int buf_len);
   static char* formatPackets(float numPkts, char *buf, u_int buf_len);
   static char* l4proto2name(u_int8_t proto);
+  static u_int8_t l4name2proto(char *name);
   static bool  isIPAddress(char *name);
   static int   setThreadAffinity(pthread_t thread, int core_id);
   static void  setThreadName(const char *name);

include/ZCCollectorInterface.h

@@ -26,7 +26,7 @@
 
 #if defined(HAVE_PF_RING) && (!defined(NTOPNG_EMBEDDED_EDITION))
 
-class ZCCollectorInterface : public ParserInterface {
+class ZCCollectorInterface : public ZMQParserInterface {
  private:
   u_int32_t cluster_id, queue_id;
   u_int32_t num_drops;

include/CollectorInterface.h → include/ZMQCollectorInterface.h

@@ -19,8 +19,8 @@
  *
  */
 
-#ifndef _COLLECTOR_INTERFACE_H_
-#define _COLLECTOR_INTERFACE_H_
+#ifndef _ZMQ_COLLECTOR_INTERFACE_H_
+#define _ZMQ_COLLECTOR_INTERFACE_H_
 
 #include "ntop_includes.h"
 
@@ -33,7 +33,7 @@ typedef struct {
   void *socket;
 } zmq_subscriber;
 
-class CollectorInterface : public ParserInterface {
+class ZMQCollectorInterface : public ZMQParserInterface {
  private:
   void *context;
   struct {
@@ -46,8 +46,8 @@ class CollectorInterface : public ParserInterface {
   zmq_subscriber subscriber[MAX_ZMQ_SUBSCRIBERS];
 
  public:
-  CollectorInterface(const char *_endpoint);
-  ~CollectorInterface();
+  ZMQCollectorInterface(const char *_endpoint);
+  ~ZMQCollectorInterface();
 
   inline const char* get_type()         { return(CONST_INTERFACE_TYPE_ZMQ);      };
   inline InterfaceType getIfType()      { return(interface_type_ZMQ); }
@@ -67,5 +67,5 @@ class CollectorInterface : public ParserInterface {
 
 #endif /* HAVE_NEDGE */
 
-#endif /* _COLLECTOR_INTERFACE_H_ */
+#endif /* _ZMQ_COLLECTOR_INTERFACE_H_ */
 

include/ZMQParserInterface.h

@@ -0,0 +1,70 @@
+/*
+ *
+ * (C) 2013-19 - ntop.org
+ *
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ */
+
+#ifndef _ZMQ_PARSER_INTERFACE_H_
+#define _ZMQ_PARSER_INTERFACE_H_
+
+#include "ntop_includes.h"
+
+class ZMQParserInterface : public ParserInterface {
+ private:
+  typedef std::pair<u_int32_t, u_int32_t> pen_value_t;
+  typedef std::map<string, pen_value_t > labels_map_t;
+  labels_map_t labels_map;
+  bool once;
+  u_int64_t zmq_initial_bytes, zmq_initial_pkts,
+    zmq_remote_initial_exported_flows;
+  ZMQ_RemoteStats *zmq_remote_stats, *zmq_remote_stats_shadow;
+#ifdef NTOPNG_PRO
+  CustomAppMaps *custom_app_maps;
+#endif
+  bool getKeyId(char *sym, u_int32_t * const pen, u_int32_t * const field) const;
+  void addMapping(const char *sym, u_int32_t num, u_int32_t pen = 0);
+  bool parsePENZeroField(Parsed_Flow * const flow, u_int32_t field, const char * const value) const;
+  bool parsePENNtopField(Parsed_Flow * const flow, u_int32_t field, const char * const value) const;
+  void parseSingleFlow(json_object *o, u_int8_t source_id, NetworkInterface *iface);
+  void setFieldMap(const ZMQ_FieldMap * const field_map) const;
+  void setFieldValueMap(const ZMQ_FieldValueMap * const field_value_map) const;
+
+  u_int8_t parseOptionFieldMap(json_object * const jo) const;
+  u_int8_t parseOptionFieldValueMap(json_object * const jo) const;
+
+ public:
+  ZMQParserInterface(const char *endpoint, const char *custom_interface_type = NULL);
+  ~ZMQParserInterface();
+
+  u_int8_t parseFlow(const char * const payload, int payload_size, u_int8_t source_id, void *data);
+  u_int8_t parseEvent(const char * const payload, int payload_size, u_int8_t source_id, void *data);
+  u_int8_t parseCounter(const char * const payload, int payload_size, u_int8_t source_id, void *data);
+  u_int8_t parseTemplate(const char * const payload, int payload_size, u_int8_t source_id, void *data);
+  u_int8_t parseOption(const char * const payload, int payload_size, u_int8_t source_id, void *data);
+
+  virtual void setRemoteStats(ZMQ_RemoteStats *zrs);
+#ifdef NTOPNG_PRO
+  virtual bool getCustomAppDetails(u_int32_t remapped_app_id, u_int32_t *const pen, u_int32_t *const app_field, u_int32_t *const app_id);
+#endif
+  u_int32_t getNumDroppedPackets() { return zmq_remote_stats ? zmq_remote_stats->sflow_pkt_sample_drops : 0; };
+  virtual void lua(lua_State* vm);
+};
+
+#endif /* _ZMQ_PARSER_INTERFACE_H_ */
+
+

include/ntop_defines.h

@@ -247,6 +247,7 @@
 #define CONST_INTERFACE_TYPE_PCAP      "pcap"
 #define CONST_INTERFACE_TYPE_PCAP_DUMP "pcap dump"
 #define CONST_INTERFACE_TYPE_ZMQ       "zmq"
+#define CONST_INTERFACE_TYPE_SYSLOG    "syslog"
 #define CONST_INTERFACE_TYPE_VLAN      "Dynamic VLAN"
 #define CONST_INTERFACE_TYPE_FLOW      "Dynamic Flow Collection"
 #define CONST_INTERFACE_TYPE_VIEW      "view"

include/ntop_includes.h

@@ -308,7 +308,10 @@ using namespace std;
 #endif
 #ifndef HAVE_NEDGE
 #include "ParserInterface.h"
-#include "CollectorInterface.h"
+#include "ZMQParserInterface.h"
+#include "ZMQCollectorInterface.h"
+#include "SyslogParserInterface.h"
+#include "SyslogCollectorInterface.h"
 #include "ZCCollectorInterface.h"
 #include "DummyInterface.h"
 #include "ExportInterface.h"

include/ntop_typedefs.h

@@ -206,7 +206,7 @@ typedef struct zmq_flow_core {
   /* Extensions used only during serialization */
   u_int16_t extn_len;
   //char extn[];
-} ZMQ_FlowCore;
+} Parsed_FlowCore;
 
 /* Handle vendor-proprietary applications.
    Must stay with 32-bit integers as, at least sonicwall, uses
@@ -218,14 +218,14 @@ typedef struct {
 } custom_app_t;
 
 typedef struct zmq_flow {
-  ZMQ_FlowCore core;
+  Parsed_FlowCore core;
   json_object *additional_fields;
   char *http_url, *http_site, *dns_query, *ssl_server_name, *bittorrent_hash;
   custom_app_t custom_app;
   /* Process Extensions */
-} ZMQ_Flow;
+} Parsed_Flow;
 
-/* IMPORTANT: whenever the ZMQ_FlowSerial is changed, nProbe must be updated too */
+/* IMPORTANT: whenever the Parsed_FlowSerial is changed, nProbe must be updated too */
 
 
 typedef struct zmq_remote_stats {
@@ -569,7 +569,8 @@ typedef enum {
   interface_type_NETFILTER,
   interface_type_DIVERT,
   interface_type_DUMMY,
-  interface_type_ZC_FLOW  
+  interface_type_ZC_FLOW,
+  interface_type_SYSLOG 
 } InterfaceType;
 
 /* Update Flow::dissectHTTP when extending the type below */

src/DummyInterface.cpp

@@ -25,7 +25,7 @@
 
 /* **************************************************** */
 
-DummyInterface::DummyInterface() : ParserInterface("dummy") {
+DummyInterface::DummyInterface() : ZMQParserInterface("dummy") {
   ntop->getTrace()->traceEvent(TRACE_NORMAL, "Initialized dummy interface");
 }
 

src/NetworkInterface.cpp

@@ -1035,7 +1035,7 @@ NetworkInterface* NetworkInterface::getSubInterface(u_int32_t criteria, bool par
 
 /* **************************************************** */
 
-void NetworkInterface::processFlow(ZMQ_Flow *zflow) {
+void NetworkInterface::processFlow(Parsed_Flow *zflow) {
   bool src2dst_direction, new_flow;
   Flow *flow;
   ndpi_protocol p;