Adds a new implementation of the event service. In this implementatio…

…n the EventCallerFlow is not used, but the event iteration will inline call the event handler. This avoids a bunch of roundtrips through the executor, and results in about 10x better performance in processing event messages.

event_handler_performance.txt

@@ -1,6 +1,9 @@
 
 
-performance data collected:
+performance data collected: 
+
+(this is for the tcs io board, with the restricted set of inputs/outputs
+enabled: 40 outputs and about 15 inputs)
 
 
 FOR COMPILATION WITH -Os
@@ -34,3 +37,18 @@ processing events with one match in the registry:
 processing events with zero match in the registry:
 1-12 msec for 100 events.
 
+
+AFTER FIRST ROUND OF OPTIMIZATION
+=================================
+
+
+processing events with 8 match in the registry:  (0100)
+24-34 msec for 100 events.
+
+processing events with one match in the registry: 
+12-20 msec for 100 events.
+
+processing events with zero match in the registry:
+2-8 msec for 100 events.
+
+

src/nmranet/EventService.cxx

@@ -19,6 +19,7 @@ namespace nmranet
 
 /*static*/
 EventService *EventService::instance = nullptr;
+static AsyncMutex event_caller_mutex;
 
 EventService::EventService(ExecutorBase *e) : Service(e)
 {
@@ -43,7 +44,7 @@ EventService::~EventService()
 
 void EventService::register_interface(If *interface)
 {
-    impl()->ownedFlows_.emplace_back(new EventIteratorFlow(
+    impl()->ownedFlows_.emplace_back(new InlineEventIteratorFlow(
         interface, this, EventService::Impl::MTI_VALUE_EVENT,
         EventService::Impl::MTI_MASK_EVENT));
     impl()->ownedFlows_.emplace_back(new EventIteratorFlow(
@@ -68,6 +69,11 @@ EventService::Impl::~Impl()
 }
 
 StateFlowBase::Action EventCallerFlow::entry()
+{
+    return allocate_and_call(STATE(perform_call), &event_caller_mutex);
+}
+
+StateFlowBase::Action EventCallerFlow::perform_call()
 {
     n_.reset(this);
     EventHandlerCall *c = message()->data();
@@ -77,6 +83,7 @@ StateFlowBase::Action EventCallerFlow::entry()
 
 StateFlowBase::Action EventCallerFlow::call_done()
 {
+    event_caller_mutex.Unlock();
     return release_and_exit();
 }
 
@@ -242,6 +249,7 @@ StateFlowBase::Action EventIteratorFlow::iterate_next()
     EventHandler *handler = iterator_->next_entry();
     if (!handler)
     {
+        no_more_matches();
         if (incomingDone_)
         {
             incomingDone_->notify();
@@ -266,6 +274,11 @@ StateFlowBase::Action EventIteratorFlow::iterate_next()
 
         return exit();
     }
+    return dispatch_event(handler);
+}
+
+StateFlowBase::Action EventIteratorFlow::dispatch_event(EventHandler* handler) 
+{
     Buffer<EventHandlerCall> *b;
     /* This could be made an asynchronous allocation. Then the pool could be
      * made fixed size. */
@@ -278,4 +291,39 @@ StateFlowBase::Action EventIteratorFlow::iterate_next()
     return wait();
 }
 
+StateFlowBase::Action InlineEventIteratorFlow::dispatch_event(EventHandler* handler)
+{
+    currentHandler_ = handler;
+    if (!holdingEventMutex_) {
+        holdingEventMutex_ = true; // will be true when we get called again
+        return allocate_and_call(STATE(perform_call), &event_caller_mutex);
+    } else {
+        return perform_call();
+    }
+}
+
+void InlineEventIteratorFlow::no_more_matches()
+{
+    if (holdingEventMutex_)
+    {
+        event_caller_mutex.Unlock();
+        holdingEventMutex_ = false;
+    }
+}
+
+StateFlowBase::Action InlineEventIteratorFlow::perform_call()
+{
+    /// @TODO (balazs.racz) we should have a different Notifiable here. The
+    /// problem is that if the Notifiable is notified inline, we want to keep
+    /// performing the iteration inline without returning to the executor. This
+    /// is not possible with the existing implementation of
+    /// BarrierNotifiable. A possible solution would be to take another child
+    /// of the barrier, and test after the call for the barrier to have exactly
+    /// one outstanding child left (no that's not a race condition); if yes,
+    /// then call immediately else notify and wait-and-call.
+    n_.reset(this);
+    (currentHandler_->*(fn_))(&eventReport_, &n_);
+    return wait_and_call(STATE(iterate_next));
+}
+
 } /* namespace nmranet */

src/nmranet/EventServiceImpl.hxx

@@ -80,6 +80,7 @@ public:
 
 private:
     virtual Action entry() OVERRIDE;
+    Action perform_call();
     Action call_done();
 
     BarrierNotifiable n_;
@@ -119,7 +120,7 @@ public:
 };
 
 /** Flow to receive incoming messages of event protocol, and dispatch them to
- * the global event handler. This flow runs on the executor of the event
+ * the registered event handler. This flow runs on the executor of the event
  * service (and not necessarily the interface). Its main job is to iterate
  * through the matching event handler and call each of them for that report. */
 class EventIteratorFlow : public IncomingMessageStateFlow
@@ -134,6 +135,12 @@ protected:
     Action iterate_next();
 
 private:
+    virtual Action dispatch_event(EventHandler* handler);
+    /// Called when there will be no more dispatch_event calls for this
+    /// iteration.
+    virtual void no_more_matches() {};
+
+protected:
     EventService *eventService_;
 
     /// Statically allocated structure for calling the event handlers from the
@@ -165,6 +172,29 @@ private:
 #endif
 };
 
+/** Flow to receive incoming messages of event protocol, and dispatch them to
+ * the registered event handler. This flow runs on the executor of the event
+ * service (and not necessarily the interface). Its main job is to iterate
+ * through the matching event handler and call each of them for that report. */
+class InlineEventIteratorFlow : public EventIteratorFlow
+{
+public:
+    InlineEventIteratorFlow(If *interface, EventService *event_service,
+                            unsigned mti_value, unsigned mti_mask) :
+        EventIteratorFlow(interface, event_service, mti_value, mti_mask) {}
+
+private:
+    Action dispatch_event(EventHandler* handler) OVERRIDE;
+    void no_more_matches() OVERRIDE;
+
+    Action perform_call();
+
+    /// True if we are already holding the event handler mutex.
+    bool holdingEventMutex_{false};
+    /// The handler we need to call.
+    EventHandler* currentHandler_{nullptr};
+};
+
 } // namespace nmranet
 
 #endif // _NMRANET_GLOBAL_EVENT_HANDLER_IMPL_